Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 66 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class AbstractFileOutStreamIntegrationTest method checkFileInUnderStorage.

/**
   * Checks the given file exists in under storage and expects its content to be an increasing
   * array of the given length.
   *
   * @param filePath path of the tmp file
   * @param fileLen length of the file
   */
protected void checkFileInUnderStorage(AlluxioURI filePath, int fileLen) throws Exception {
    URIStatus status = mFileSystem.getStatus(filePath);
    String checkpointPath = status.getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(checkpointPath);
    try (InputStream is = ufs.open(checkpointPath)) {
        byte[] res = new byte[(int) status.getLength()];
        String underFSClass = UnderFileSystemCluster.getUnderFSClass();
        if ((LocalMiniDFSCluster.class.getName().equals(underFSClass)) && 0 == res.length) {
            // Returns -1 for zero-sized byte array to indicate no more bytes available here.
            Assert.assertEquals(-1, is.read(res));
        } else {
            Assert.assertEquals((int) status.getLength(), is.read(res));
        }
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(fileLen, res));
    }
}

Example 67 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method checkConsistencyInternal.

/**
   * Checks if a path is consistent between Alluxio and the underlying storage.
   * <p>
   * A path without a backing under storage is always consistent.
   * <p>
   * A not persisted path is considered consistent if:
   * 1. It does not shadow an object in the underlying storage.
   * <p>
   * A persisted path is considered consistent if:
   * 1. An equivalent object exists for its under storage path.
   * 2. The metadata of the Alluxio and under storage object are equal.
   *
   * @param inode the inode to check
   * @param path the current path associated with the inode
   * @return true if the path is consistent, false otherwise
   * @throws FileDoesNotExistException if the path cannot be found in the Alluxio inode tree
   * @throws InvalidPathException if the path is not well formed
   */
private boolean checkConsistencyInternal(Inode inode, AlluxioURI path) throws FileDoesNotExistException, InvalidPathException, IOException {
    MountTable.Resolution resolution = mMountTable.resolve(path);
    UnderFileSystem ufs = resolution.getUfs();
    String ufsPath = resolution.getUri().getPath();
    if (ufs == null) {
        return true;
    }
    if (!inode.isPersisted()) {
        return !ufs.exists(ufsPath);
    }
    // TODO(calvin): Evaluate which other metadata fields should be validated.
    if (inode.isDirectory()) {
        return ufs.isDirectory(ufsPath);
    } else {
        InodeFile file = (InodeFile) inode;
        return ufs.isFile(ufsPath) && ufs.getFileSize(ufsPath) == file.getLength();
    }
}

Example 68 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method generateFileBlockInfo.

/**
   * Generates a {@link FileBlockInfo} object from internal metadata. This adds file information to
   * the block, such as the file offset, and additional UFS locations for the block.
   *
   * @param inodePath the file the block is a part of
   * @param blockInfo the {@link BlockInfo} to generate the {@link FileBlockInfo} from
   * @return a new {@link FileBlockInfo} for the block
   * @throws InvalidPathException if the mount table is not able to resolve the file
   */
private FileBlockInfo generateFileBlockInfo(LockedInodePath inodePath, BlockInfo blockInfo) throws InvalidPathException, FileDoesNotExistException {
    InodeFile file = inodePath.getInodeFile();
    FileBlockInfo fileBlockInfo = new FileBlockInfo();
    fileBlockInfo.setBlockInfo(blockInfo);
    fileBlockInfo.setUfsLocations(new ArrayList<String>());
    // The sequence number part of the block id is the block index.
    long offset = file.getBlockSizeBytes() * BlockId.getSequenceNumber(blockInfo.getBlockId());
    fileBlockInfo.setOffset(offset);
    if (fileBlockInfo.getBlockInfo().getLocations().isEmpty() && file.isPersisted()) {
        // No alluxio locations, but there is a checkpoint in the under storage system. Add the
        // locations from the under storage system.
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        String ufsUri = resolution.getUri().toString();
        UnderFileSystem ufs = resolution.getUfs();
        List<String> locs;
        try {
            locs = ufs.getFileLocations(ufsUri, FileLocationOptions.defaults().setOffset(fileBlockInfo.getOffset()));
        } catch (IOException e) {
            return fileBlockInfo;
        }
        if (locs != null) {
            for (String loc : locs) {
                fileBlockInfo.getUfsLocations().add(loc);
            }
        }
    }
    return fileBlockInfo;
}

Example 69 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method setAttributeInternal.

/**
   * @param inodePath the {@link LockedInodePath} to use
   * @param replayed whether the operation is a result of replaying the journal
   * @param opTimeMs the operation time (in milliseconds)
   * @param options the method options
   * @return list of inodes which were marked as persisted
   * @throws FileDoesNotExistException if the file does not exist
   * @throws InvalidPathException if the file path corresponding to the file id is invalid
   * @throws AccessControlException if failed to set permission
   */
private List<Inode<?>> setAttributeInternal(LockedInodePath inodePath, boolean replayed, long opTimeMs, SetAttributeOptions options) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    List<Inode<?>> persistedInodes = Collections.emptyList();
    Inode<?> inode = inodePath.getInode();
    if (options.getPinned() != null) {
        mInodeTree.setPinned(inodePath, options.getPinned(), opTimeMs);
        inode.setLastModificationTimeMs(opTimeMs);
    }
    if (options.getTtl() != null) {
        long ttl = options.getTtl();
        if (inode.getTtl() != ttl) {
            mTtlBuckets.remove(inode);
            inode.setTtl(ttl);
            mTtlBuckets.insert(inode);
            inode.setLastModificationTimeMs(opTimeMs);
            inode.setTtlAction(options.getTtlAction());
        }
    }
    if (options.getPersisted() != null) {
        Preconditions.checkArgument(inode.isFile(), PreconditionMessage.PERSIST_ONLY_FOR_FILE);
        Preconditions.checkArgument(((InodeFile) inode).isCompleted(), PreconditionMessage.FILE_TO_PERSIST_MUST_BE_COMPLETE);
        InodeFile file = (InodeFile) inode;
        // TODO(manugoyal) figure out valid behavior in the un-persist case
        Preconditions.checkArgument(options.getPersisted(), PreconditionMessage.ERR_SET_STATE_UNPERSIST);
        if (!file.isPersisted()) {
            file.setPersistenceState(PersistenceState.PERSISTED);
            persistedInodes = propagatePersistedInternal(inodePath, false);
            file.setLastModificationTimeMs(opTimeMs);
            Metrics.FILES_PERSISTED.inc();
        }
    }
    boolean ownerGroupChanged = (options.getOwner() != null) || (options.getGroup() != null);
    boolean modeChanged = (options.getMode() != Constants.INVALID_MODE);
    // If the file is persisted in UFS, also update corresponding owner/group/permission.
    if ((ownerGroupChanged || modeChanged) && !replayed && inode.isPersisted()) {
        if ((inode instanceof InodeFile) && !((InodeFile) inode).isCompleted()) {
            LOG.debug("Alluxio does not propagate chown/chgrp/chmod to UFS for incomplete files.");
        } else {
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            String ufsUri = resolution.getUri().toString();
            if (UnderFileSystemUtils.isObjectStorage(ufsUri)) {
                LOG.warn("setOwner/setMode is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
            } else {
                UnderFileSystem ufs = resolution.getUfs();
                if (ownerGroupChanged) {
                    try {
                        String owner = options.getOwner() != null ? options.getOwner() : inode.getOwner();
                        String group = options.getGroup() != null ? options.getGroup() : inode.getGroup();
                        ufs.setOwner(ufsUri, owner, group);
                    } catch (IOException e) {
                        throw new AccessControlException("Could not setOwner for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                    }
                }
                if (modeChanged) {
                    try {
                        ufs.setMode(ufsUri, options.getMode());
                    } catch (IOException e) {
                        throw new AccessControlException("Could not setMode for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                    }
                }
            }
        }
    }
    // Only commit the set permission to inode after the propagation to UFS succeeded.
    if (options.getOwner() != null) {
        inode.setOwner(options.getOwner());
    }
    if (options.getGroup() != null) {
        inode.setGroup(options.getGroup());
    }
    if (modeChanged) {
        inode.setMode(options.getMode());
    }
    return persistedInodes;
}

Example 70 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method loadFileMetadataAndJournal.

/**
   * Loads metadata for the file identified by the given path from UFS into Alluxio.
   *
   * @param inodePath the path for which metadata should be loaded
   * @param resolution the UFS resolution of path
   * @param options the load metadata options
   * @param journalContext the journal context
   * @throws BlockInfoException if an invalid block size is encountered
   * @throws FileDoesNotExistException if there is no UFS path
   * @throws InvalidPathException if invalid path is encountered
   * @throws AccessControlException if permission checking fails or permission setting fails
   * @throws FileAlreadyCompletedException if the file is already completed
   * @throws InvalidFileSizeException if invalid file size is encountered
   * @throws IOException if an I/O error occurs
   */
private void loadFileMetadataAndJournal(LockedInodePath inodePath, MountTable.Resolution resolution, LoadMetadataOptions options, JournalContext journalContext) throws BlockInfoException, FileDoesNotExistException, InvalidPathException, AccessControlException, FileAlreadyCompletedException, InvalidFileSizeException, IOException {
    if (inodePath.fullPathExists()) {
        return;
    }
    AlluxioURI ufsUri = resolution.getUri();
    UnderFileSystem ufs = resolution.getUfs();
    long ufsBlockSizeByte = ufs.getBlockSizeByte(ufsUri.toString());
    long ufsLength = ufs.getFileSize(ufsUri.toString());
    // Metadata loaded from UFS has no TTL set.
    CreateFileOptions createFileOptions = CreateFileOptions.defaults().setBlockSizeBytes(ufsBlockSizeByte).setRecursive(options.isCreateAncestors()).setMetadataLoad(true).setPersisted(true);
    String ufsOwner = ufs.getOwner(ufsUri.toString());
    String ufsGroup = ufs.getGroup(ufsUri.toString());
    short ufsMode = ufs.getMode(ufsUri.toString());
    Mode mode = new Mode(ufsMode);
    if (resolution.getShared()) {
        mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits()));
    }
    createFileOptions = createFileOptions.setOwner(ufsOwner).setGroup(ufsGroup).setMode(mode);
    try {
        createFileAndJournal(inodePath, createFileOptions, journalContext);
        CompleteFileOptions completeOptions = CompleteFileOptions.defaults().setUfsLength(ufsLength);
        completeFileAndJournal(inodePath, completeOptions, journalContext);
    } catch (FileAlreadyExistsException e) {
        LOG.error("FileAlreadyExistsException seen unexpectedly.", e);
        throw new RuntimeException(e);
    }
}