Examples with DescendantType alluxio.file.options.DescendantType used on opensource projects

Example 1 with DescendantType

use of alluxio.file.options.DescendantType in project alluxio by Alluxio.

the class DefaultFileSystemMaster method listStatusInternal.

/**
 * Lists the status of the path in {@link LockedInodePath}, possibly recursively depending on the
 * descendantType. The result is returned via a list specified by statusList, in postorder
 * traversal order.
 *
 * @param context call context
 * @param rpcContext the context for the RPC call
 * @param currInodePath the inode path to find the status
 * @param auditContext the audit context to return any access exceptions
 * @param descendantType if the currInodePath is a directory, how many levels of its descendant
 *        should be returned
 * @param resultStream the stream to receive individual results
 * @param depth internal use field for tracking depth relative to root item
 */
private void listStatusInternal(ListStatusContext context, RpcContext rpcContext, LockedInodePath currInodePath, AuditContext auditContext, DescendantType descendantType, ResultStream<FileInfo> resultStream, int depth, Counter counter) throws FileDoesNotExistException, UnavailableException, AccessControlException, InvalidPathException {
    rpcContext.throwIfCancelled();
    Inode inode = currInodePath.getInode();
    if (inode.isDirectory() && descendantType != DescendantType.NONE) {
        try {
            // TODO(david): Return the error message when we do not have permission
            mPermissionChecker.checkPermission(Mode.Bits.EXECUTE, currInodePath);
        } catch (AccessControlException e) {
            auditContext.setAllowed(false);
            if (descendantType == DescendantType.ALL) {
                return;
            } else {
                throw e;
            }
        }
        mAccessTimeUpdater.updateAccessTime(rpcContext.getJournalContext(), inode, CommonUtils.getCurrentMs());
        DescendantType nextDescendantType = (descendantType == DescendantType.ALL) ? DescendantType.ALL : DescendantType.NONE;
        // This is to generate a parsed child path components to be passed to lockChildPath
        String[] childComponentsHint = null;
        for (Inode child : mInodeStore.getChildren(inode.asDirectory())) {
            if (childComponentsHint == null) {
                String[] parentComponents = PathUtils.getPathComponents(currInodePath.getUri().getPath());
                childComponentsHint = new String[parentComponents.length + 1];
                System.arraycopy(parentComponents, 0, childComponentsHint, 0, parentComponents.length);
            }
            // TODO(david): Make extending InodePath more efficient
            childComponentsHint[childComponentsHint.length - 1] = child.getName();
            try (LockedInodePath childInodePath = currInodePath.lockChild(child, LockPattern.READ, childComponentsHint)) {
                listStatusInternal(context, rpcContext, childInodePath, auditContext, nextDescendantType, resultStream, depth + 1, counter);
            } catch (InvalidPathException | FileDoesNotExistException e) {
                LOG.debug("Path \"{}\" is invalid, has been ignored.", PathUtils.concatPath("/", childComponentsHint));
            }
        }
    }
    // Listing a directory should not emit item for the directory itself.
    if (depth != 0 || inode.isFile()) {
        resultStream.submit(getFileInfoInternal(currInodePath, counter));
    }
}

Example 2 with DescendantType

use of alluxio.file.options.DescendantType in project alluxio by Alluxio.

the class DefaultFileSystemMaster method listStatus.

@Override
public void listStatus(AlluxioURI path, ListStatusContext context, ResultStream<FileInfo> resultStream) throws AccessControlException, FileDoesNotExistException, InvalidPathException, IOException {
    Metrics.GET_FILE_INFO_OPS.inc();
    LockingScheme lockingScheme = new LockingScheme(path, LockPattern.READ, false);
    boolean ufsAccessed = false;
    try (RpcContext rpcContext = createRpcContext(context);
        FileSystemMasterAuditContext auditContext = createAuditContext("listStatus", path, null, null)) {
        DescendantType descendantType = context.getOptions().getRecursive() ? DescendantType.ALL : DescendantType.ONE;
        if (!syncMetadata(rpcContext, path, context.getOptions().getCommonOptions(), descendantType, auditContext, LockedInodePath::getInodeOrNull, (inodePath, permChecker) -> permChecker.checkPermission(Mode.Bits.READ, inodePath), false).equals(NOT_NEEDED)) {
            // If synced, do not load metadata.
            context.getOptions().setLoadMetadataType(LoadMetadataPType.NEVER);
            ufsAccessed = true;
        }
        /*
      See the comments in #getFileIdInternal for an explanation on why the loop here is required.
       */
        DescendantType loadDescendantType;
        if (context.getOptions().getLoadMetadataType() == LoadMetadataPType.NEVER) {
            loadDescendantType = DescendantType.NONE;
        } else if (context.getOptions().getRecursive()) {
            loadDescendantType = DescendantType.ALL;
        } else {
            loadDescendantType = DescendantType.ONE;
        }
        // load metadata for 1 level of descendants, or all descendants if recursive
        LoadMetadataContext loadMetadataContext = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCreateAncestors(true).setLoadType(context.getOptions().getLoadMetadataType()).setLoadDescendantType(GrpcUtils.toProto(loadDescendantType)).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction())));
        boolean loadMetadata = false;
        boolean run = true;
        while (run) {
            run = false;
            if (loadMetadata) {
                loadMetadataIfNotExist(rpcContext, path, loadMetadataContext, false);
                ufsAccessed = true;
            }
            // We just synced; the new lock pattern should not sync.
            try (LockedInodePath inodePath = mInodeTree.lockInodePath(lockingScheme)) {
                auditContext.setSrcInode(inodePath.getInodeOrNull());
                try {
                    mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
                } catch (AccessControlException e) {
                    auditContext.setAllowed(false);
                    throw e;
                }
                if (!loadMetadata) {
                    Inode inode;
                    boolean isLoaded = true;
                    if (inodePath.fullPathExists()) {
                        inode = inodePath.getInode();
                        if (inode.isDirectory() && context.getOptions().getLoadMetadataType() != LoadMetadataPType.ALWAYS) {
                            InodeDirectory inodeDirectory = inode.asDirectory();
                            isLoaded = inodeDirectory.isDirectChildrenLoaded();
                            if (context.getOptions().getRecursive()) {
                                isLoaded = areDescendantsLoaded(inodeDirectory);
                            }
                            if (isLoaded) {
                                // no need to load again.
                                loadMetadataContext.getOptions().setLoadDescendantType(LoadDescendantPType.NONE);
                            }
                        }
                    } else {
                        checkLoadMetadataOptions(context.getOptions().getLoadMetadataType(), inodePath.getUri());
                    }
                    if (shouldLoadMetadataIfNotExists(inodePath, loadMetadataContext)) {
                        loadMetadata = true;
                        run = true;
                        continue;
                    }
                }
                ensureFullPathAndUpdateCache(inodePath);
                auditContext.setSrcInode(inodePath.getInode());
                MountTable.Resolution resolution;
                if (!context.getOptions().hasLoadMetadataOnly() || !context.getOptions().getLoadMetadataOnly()) {
                    DescendantType descendantTypeForListStatus = (context.getOptions().getRecursive()) ? DescendantType.ALL : DescendantType.ONE;
                    try {
                        resolution = mMountTable.resolve(path);
                    } catch (InvalidPathException e) {
                        throw new FileDoesNotExistException(e.getMessage(), e);
                    }
                    listStatusInternal(context, rpcContext, inodePath, auditContext, descendantTypeForListStatus, resultStream, 0, Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.GET_FILE_INFO));
                    if (!ufsAccessed) {
                        Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.LIST_STATUS).inc();
                    }
                }
                auditContext.setSucceeded(true);
                Metrics.FILE_INFOS_GOT.inc();
            }
        }
    }
}

Example 3 with DescendantType

use of alluxio.file.options.DescendantType in project alluxio by Alluxio.

the class DefaultFileSystemMaster method loadMetadataIfNotExist.

/**
 * Loads metadata for the path if it is (non-existing || load direct children is set).
 *
 * See {@link #shouldLoadMetadataIfNotExists(LockedInodePath, LoadMetadataContext)}.
 *
 * @param rpcContext the rpc context
 * @param path the path to load metadata for
 * @param context the {@link LoadMetadataContext}
 * @param isGetFileInfo whether this is loading for a {@link #getFileInfo} call
 */
private void loadMetadataIfNotExist(RpcContext rpcContext, AlluxioURI path, LoadMetadataContext context, boolean isGetFileInfo) throws InvalidPathException, AccessControlException {
    DescendantType syncDescendantType = GrpcUtils.fromProto(context.getOptions().getLoadDescendantType());
    FileSystemMasterCommonPOptions commonOptions = context.getOptions().getCommonOptions();
    boolean loadAlways = context.getOptions().hasLoadType() && (context.getOptions().getLoadType().equals(LoadMetadataPType.ALWAYS));
    // load metadata only and force sync
    InodeSyncStream sync = new InodeSyncStream(new LockingScheme(path, LockPattern.READ, false), this, rpcContext, syncDescendantType, commonOptions, isGetFileInfo, true, true, loadAlways);
    if (sync.sync().equals(FAILED)) {
        LOG.debug("Failed to load metadata for path from UFS: {}", path);
    }
}

Example 4 with DescendantType

use of alluxio.file.options.DescendantType in project alluxio by Alluxio.

the class InodeSyncStream method loadMetadataForPath.

private void loadMetadataForPath(LockedInodePath inodePath) throws InvalidPathException, AccessControlException, IOException, FileDoesNotExistException, FileAlreadyCompletedException, InvalidFileSizeException, BlockInfoException {
    UfsStatus status = mStatusCache.fetchStatusIfAbsent(inodePath.getUri(), mMountTable);
    DescendantType descendantType = mDescendantType;
    // do not load the subdirectory
    if (descendantType.equals(DescendantType.ONE) && !inodePath.getUri().equals(mRootScheme.getPath())) {
        descendantType = DescendantType.NONE;
    }
    LoadMetadataContext ctx = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCommonOptions(NO_TTL_OPTION).setCreateAncestors(true).setLoadDescendantType(GrpcUtils.toProto(descendantType))).setUfsStatus(status);
    loadMetadata(inodePath, ctx);
}