Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 26 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method setAttributeSingleFile.

/**
 * @param inodePath the {@link LockedInodePath} to use
 * @param updateUfs whether to update the UFS with the attribute change
 * @param opTimeMs the operation time (in milliseconds)
 * @param context the method context
 */
protected void setAttributeSingleFile(RpcContext rpcContext, LockedInodePath inodePath, boolean updateUfs, long opTimeMs, SetAttributeContext context) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    Inode inode = inodePath.getInode();
    SetAttributePOptions.Builder protoOptions = context.getOptions();
    if (protoOptions.hasPinned()) {
        mInodeTree.setPinned(rpcContext, inodePath, context.getOptions().getPinned(), context.getOptions().getPinnedMediaList(), opTimeMs);
    }
    UpdateInodeEntry.Builder entry = UpdateInodeEntry.newBuilder().setId(inode.getId());
    if (protoOptions.hasReplicationMax() || protoOptions.hasReplicationMin()) {
        Integer replicationMax = protoOptions.hasReplicationMax() ? protoOptions.getReplicationMax() : null;
        Integer replicationMin = protoOptions.hasReplicationMin() ? protoOptions.getReplicationMin() : null;
        mInodeTree.setReplication(rpcContext, inodePath, replicationMax, replicationMin, opTimeMs);
    }
    // protoOptions may not have both fields set
    if (protoOptions.hasCommonOptions()) {
        FileSystemMasterCommonPOptions commonOpts = protoOptions.getCommonOptions();
        TtlAction action = commonOpts.hasTtlAction() ? commonOpts.getTtlAction() : null;
        Long ttl = commonOpts.hasTtl() ? commonOpts.getTtl() : null;
        boolean modified = false;
        if (ttl != null && inode.getTtl() != ttl) {
            entry.setTtl(ttl);
            modified = true;
        }
        if (action != null && inode.getTtlAction() != action) {
            entry.setTtlAction(ProtobufUtils.toProtobuf(action));
            modified = true;
        }
        if (modified) {
            entry.setLastModificationTimeMs(opTimeMs);
        }
    }
    if (protoOptions.hasPersisted()) {
        Preconditions.checkArgument(inode.isFile(), PreconditionMessage.PERSIST_ONLY_FOR_FILE);
        Preconditions.checkArgument(inode.asFile().isCompleted(), PreconditionMessage.FILE_TO_PERSIST_MUST_BE_COMPLETE);
        // TODO(manugoyal) figure out valid behavior in the un-persist case
        Preconditions.checkArgument(protoOptions.getPersisted(), PreconditionMessage.ERR_SET_STATE_UNPERSIST);
        if (!inode.asFile().isPersisted()) {
            entry.setPersistenceState(PersistenceState.PERSISTED.name());
            entry.setLastModificationTimeMs(context.getOperationTimeMs());
            propagatePersistedInternal(rpcContext, inodePath);
            Metrics.FILES_PERSISTED.inc();
        }
    }
    boolean ownerGroupChanged = (protoOptions.hasOwner()) || (protoOptions.hasGroup());
    boolean modeChanged = protoOptions.hasMode();
    // If the file is persisted in UFS, also update corresponding owner/group/permission.
    if ((ownerGroupChanged || modeChanged) && updateUfs && inode.isPersisted()) {
        if ((inode instanceof InodeFile) && !inode.asFile().isCompleted()) {
            LOG.debug("Alluxio does not propagate chown/chgrp/chmod to UFS for incomplete files.");
        } else {
            checkUfsMode(inodePath.getUri(), OperationType.WRITE);
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            String ufsUri = resolution.getUri().toString();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                if (ufs.isObjectStorage()) {
                    LOG.debug("setOwner/setMode is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
                } else {
                    String owner = null;
                    String group = null;
                    String mode = null;
                    if (ownerGroupChanged) {
                        try {
                            owner = protoOptions.getOwner() != null ? protoOptions.getOwner() : inode.getOwner();
                            group = protoOptions.getGroup() != null ? protoOptions.getGroup() : inode.getGroup();
                            ufs.setOwner(ufsUri, owner, group);
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setOwner for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    if (modeChanged) {
                        try {
                            mode = String.valueOf(protoOptions.getMode());
                            ufs.setMode(ufsUri, ModeUtils.protoToShort(protoOptions.getMode()));
                        } catch (IOException e) {
                            throw new AccessControlException("Could not setMode for UFS file " + ufsUri + " . Aborting the setAttribute operation in Alluxio.", e);
                        }
                    }
                    // Retrieve the ufs fingerprint after the ufs changes.
                    String existingFingerprint = inode.getUfsFingerprint();
                    if (!existingFingerprint.equals(Constants.INVALID_UFS_FINGERPRINT)) {
                        // Update existing fingerprint, since contents did not change
                        Fingerprint fp = Fingerprint.parse(existingFingerprint);
                        fp.putTag(Fingerprint.Tag.OWNER, owner);
                        fp.putTag(Fingerprint.Tag.GROUP, group);
                        fp.putTag(Fingerprint.Tag.MODE, mode);
                        context.setUfsFingerprint(fp.serialize());
                    } else {
                        // Need to retrieve the fingerprint from ufs.
                        context.setUfsFingerprint(ufs.getFingerprint(ufsUri));
                    }
                }
            }
        }
    }
    if (!context.getUfsFingerprint().equals(Constants.INVALID_UFS_FINGERPRINT)) {
        entry.setUfsFingerprint(context.getUfsFingerprint());
    }
    // Only commit the set permission to inode after the propagation to UFS succeeded.
    if (protoOptions.hasOwner()) {
        entry.setOwner(protoOptions.getOwner());
    }
    if (protoOptions.hasGroup()) {
        entry.setGroup(protoOptions.getGroup());
    }
    if (modeChanged) {
        entry.setMode(ModeUtils.protoToShort(protoOptions.getMode()));
    }
    mInodeTree.updateInode(rpcContext, entry.build());
}

Example 27 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method checkConsistencyRecursive.

private void checkConsistencyRecursive(LockedInodePath inodePath, List<AlluxioURI> inconsistentUris, boolean assertInconsistent, boolean metadataSynced) throws IOException, FileDoesNotExistException {
    Inode inode = inodePath.getInode();
    try {
        if (assertInconsistent || !checkConsistencyInternal(inodePath)) {
            inconsistentUris.add(inodePath.getUri());
            // If a dir in Alluxio is inconsistent with underlying storage,
            // we can assert the children is inconsistent.
            // If a file is inconsistent, please ignore this parameter cause it has no child node.
            assertInconsistent = true;
        }
        if (inode.isDirectory()) {
            InodeDirectory inodeDir = inode.asDirectory();
            Iterable<? extends Inode> children = mInodeStore.getChildren(inodeDir);
            for (Inode child : children) {
                try (LockedInodePath childPath = inodePath.lockChild(child, LockPattern.READ)) {
                    checkConsistencyRecursive(childPath, inconsistentUris, assertInconsistent, metadataSynced);
                }
            }
            // if the metadata has already been synced, then we could skip it.
            if (metadataSynced) {
                return;
            }
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            UfsStatus[] statuses;
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                String ufsPath = resolution.getUri().getPath();
                statuses = ufs.listStatus(ufsPath);
            }
            if (statuses != null) {
                HashSet<String> alluxioFileNames = Streams.stream(children).map(Inode::getName).collect(Collectors.toCollection(HashSet::new));
                Arrays.stream(statuses).forEach(status -> {
                    if (!alluxioFileNames.contains(status.getName())) {
                        inconsistentUris.add(inodePath.getUri().join(status.getName()));
                    }
                });
            }
        }
    } catch (InvalidPathException e) {
        LOG.debug("Path \"{}\" is invalid, has been ignored.", PathUtils.concatPath(inodePath.getUri().getPath()));
    }
}

Example 28 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method checkUfsMode.

/**
 * Check if the specified operation type is allowed to the ufs.
 *
 * @param alluxioPath the Alluxio path
 * @param opType the operation type
 */
private void checkUfsMode(AlluxioURI alluxioPath, OperationType opType) throws AccessControlException, InvalidPathException {
    MountTable.Resolution resolution = mMountTable.resolve(alluxioPath);
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        UfsMode ufsMode = ufs.getOperationMode(mUfsManager.getPhysicalUfsState(ufs.getPhysicalStores()));
        switch(ufsMode) {
            case NO_ACCESS:
                throw new AccessControlException(ExceptionMessage.UFS_OP_NOT_ALLOWED.getMessage(opType, resolution.getUri(), UfsMode.NO_ACCESS));
            case READ_ONLY:
                if (opType == OperationType.WRITE) {
                    throw new AccessControlException(ExceptionMessage.UFS_OP_NOT_ALLOWED.getMessage(opType, resolution.getUri(), UfsMode.READ_ONLY));
                }
                break;
            default:
                // All operations are allowed
                break;
        }
    }
}

Example 29 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method completeFileInternal.

/**
 * Completes a file. After a file is completed, it cannot be written to.
 *
 * @param rpcContext the rpc context
 * @param inodePath the {@link LockedInodePath} to complete
 * @param context the method context
 */
void completeFileInternal(RpcContext rpcContext, LockedInodePath inodePath, CompleteFileContext context) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException, UnavailableException {
    Inode inode = inodePath.getInode();
    if (!inode.isFile()) {
        throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(inodePath.getUri()));
    }
    InodeFile fileInode = inode.asFile();
    List<Long> blockIdList = fileInode.getBlockIds();
    List<BlockInfo> blockInfoList = mBlockMaster.getBlockInfoList(blockIdList);
    if (!fileInode.isPersisted() && blockInfoList.size() != blockIdList.size()) {
        throw new BlockInfoException("Cannot complete a file without all the blocks committed");
    }
    // Iterate over all file blocks committed to Alluxio, computing the length and verify that all
    // the blocks (except the last one) is the same size as the file block size.
    long inAlluxioLength = 0;
    long fileBlockSize = fileInode.getBlockSizeBytes();
    for (int i = 0; i < blockInfoList.size(); i++) {
        BlockInfo blockInfo = blockInfoList.get(i);
        inAlluxioLength += blockInfo.getLength();
        if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) {
            throw new BlockInfoException("Block index " + i + " has a block size smaller than the file block size (" + fileInode.getBlockSizeBytes() + ")");
        }
    }
    // If the file is persisted, its length is determined by UFS. Otherwise, its length is
    // determined by its size in Alluxio.
    long length = fileInode.isPersisted() ? context.getOptions().getUfsLength() : inAlluxioLength;
    String ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
    if (fileInode.isPersisted()) {
        UfsStatus ufsStatus = context.getUfsStatus();
        // Retrieve the UFS fingerprint for this file.
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        AlluxioURI resolvedUri = resolution.getUri();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            if (ufsStatus == null) {
                ufsFingerprint = ufs.getFingerprint(resolvedUri.toString());
            } else {
                ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
            }
        }
    }
    completeFileInternal(rpcContext, inodePath, length, context.getOperationTimeMs(), ufsFingerprint);
}

Example 30 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method createDirectoryInternal.

/**
 * Implementation of directory creation for a given path.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path of the directory
 * @param context method context
 * @return a list of created inodes
 */
List<Inode> createDirectoryInternal(RpcContext rpcContext, LockedInodePath inodePath, CreateDirectoryContext context) throws InvalidPathException, FileAlreadyExistsException, IOException, FileDoesNotExistException {
    Preconditions.checkState(inodePath.getLockPattern() == LockPattern.WRITE_EDGE);
    try {
        List<Inode> createResult = mInodeTree.createPath(rpcContext, inodePath, context);
        InodeDirectory inodeDirectory = inodePath.getInode().asDirectory();
        String ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
        if (inodeDirectory.isPersisted()) {
            UfsStatus ufsStatus = context.getUfsStatus();
            // Retrieve the UFS fingerprint for this file.
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            AlluxioURI resolvedUri = resolution.getUri();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                if (ufsStatus == null) {
                    ufsFingerprint = ufs.getFingerprint(resolvedUri.toString());
                } else {
                    ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
                }
            }
        }
        mInodeTree.updateInode(rpcContext, UpdateInodeEntry.newBuilder().setId(inodeDirectory.getId()).setUfsFingerprint(ufsFingerprint).build());
        if (context.isPersisted()) {
            // The path exists in UFS, so it is no longer absent.
            mUfsAbsentPathCache.processExisting(inodePath.getUri());
        }
        Metrics.DIRECTORIES_CREATED.inc();
        return createResult;
    } catch (BlockInfoException e) {
        // happen.
        throw new RuntimeException(e);
    }
}