Examples with LoadMetadataContext alluxio.master.file.contexts.LoadMetadataContext used on opensource projects

Example 1 with LoadMetadataContext

use of alluxio.master.file.contexts.LoadMetadataContext in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getFileInfo.

@Override
public FileInfo getFileInfo(AlluxioURI path, GetStatusContext context) throws FileDoesNotExistException, InvalidPathException, AccessControlException, IOException {
    Metrics.GET_FILE_INFO_OPS.inc();
    boolean ufsAccessed = false;
    long opTimeMs = System.currentTimeMillis();
    try (RpcContext rpcContext = createRpcContext(context);
        FileSystemMasterAuditContext auditContext = createAuditContext("getFileInfo", path, null, null)) {
        if (!syncMetadata(rpcContext, path, context.getOptions().getCommonOptions(), DescendantType.ONE, auditContext, LockedInodePath::getInodeOrNull, (inodePath, permChecker) -> permChecker.checkPermission(Mode.Bits.READ, inodePath), true).equals(NOT_NEEDED)) {
            // If synced, do not load metadata.
            context.getOptions().setLoadMetadataType(LoadMetadataPType.NEVER);
            ufsAccessed = true;
        }
        LoadMetadataContext lmCtx = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCreateAncestors(true).setLoadType(context.getOptions().getLoadMetadataType()).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction())));
        /*
      See the comments in #getFileIdInternal for an explanation on why the loop here is required.
       */
        boolean run = true;
        boolean loadMetadata = false;
        FileInfo ret = null;
        while (run) {
            run = false;
            if (loadMetadata) {
                checkLoadMetadataOptions(context.getOptions().getLoadMetadataType(), path);
                loadMetadataIfNotExist(rpcContext, path, lmCtx, true);
                ufsAccessed = true;
            }
            LockingScheme lockingScheme = new LockingScheme(path, LockPattern.READ, false);
            try (LockedInodePath inodePath = mInodeTree.lockInodePath(lockingScheme)) {
                auditContext.setSrcInode(inodePath.getInodeOrNull());
                try {
                    mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
                } catch (AccessControlException e) {
                    auditContext.setAllowed(false);
                    throw e;
                }
                if (!loadMetadata && shouldLoadMetadataIfNotExists(inodePath, lmCtx)) {
                    loadMetadata = true;
                    run = true;
                    continue;
                }
                ensureFullPathAndUpdateCache(inodePath);
                FileInfo fileInfo = getFileInfoInternal(inodePath);
                if (!fileInfo.isFolder() && (!fileInfo.isCompleted())) {
                    LOG.debug("File {} is not yet completed. getStatus will see incomplete metadata.", fileInfo.getPath());
                }
                if (ufsAccessed) {
                    MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
                    Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.GET_FILE_INFO).dec();
                }
                Mode.Bits accessMode = Mode.Bits.fromProto(context.getOptions().getAccessMode());
                if (context.getOptions().getUpdateTimestamps() && context.getOptions().hasAccessMode() && (accessMode.imply(Mode.Bits.READ) || accessMode.imply(Mode.Bits.WRITE))) {
                    mAccessTimeUpdater.updateAccessTime(rpcContext.getJournalContext(), inodePath.getInode(), opTimeMs);
                }
                auditContext.setSrcInode(inodePath.getInode()).setSucceeded(true);
                ret = fileInfo;
            }
        }
        return ret;
    }
}

Example 2 with LoadMetadataContext

use of alluxio.master.file.contexts.LoadMetadataContext in project alluxio by Alluxio.

the class DefaultFileSystemMaster method listStatus.

@Override
public void listStatus(AlluxioURI path, ListStatusContext context, ResultStream<FileInfo> resultStream) throws AccessControlException, FileDoesNotExistException, InvalidPathException, IOException {
    Metrics.GET_FILE_INFO_OPS.inc();
    LockingScheme lockingScheme = new LockingScheme(path, LockPattern.READ, false);
    boolean ufsAccessed = false;
    try (RpcContext rpcContext = createRpcContext(context);
        FileSystemMasterAuditContext auditContext = createAuditContext("listStatus", path, null, null)) {
        DescendantType descendantType = context.getOptions().getRecursive() ? DescendantType.ALL : DescendantType.ONE;
        if (!syncMetadata(rpcContext, path, context.getOptions().getCommonOptions(), descendantType, auditContext, LockedInodePath::getInodeOrNull, (inodePath, permChecker) -> permChecker.checkPermission(Mode.Bits.READ, inodePath), false).equals(NOT_NEEDED)) {
            // If synced, do not load metadata.
            context.getOptions().setLoadMetadataType(LoadMetadataPType.NEVER);
            ufsAccessed = true;
        }
        /*
      See the comments in #getFileIdInternal for an explanation on why the loop here is required.
       */
        DescendantType loadDescendantType;
        if (context.getOptions().getLoadMetadataType() == LoadMetadataPType.NEVER) {
            loadDescendantType = DescendantType.NONE;
        } else if (context.getOptions().getRecursive()) {
            loadDescendantType = DescendantType.ALL;
        } else {
            loadDescendantType = DescendantType.ONE;
        }
        // load metadata for 1 level of descendants, or all descendants if recursive
        LoadMetadataContext loadMetadataContext = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCreateAncestors(true).setLoadType(context.getOptions().getLoadMetadataType()).setLoadDescendantType(GrpcUtils.toProto(loadDescendantType)).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction())));
        boolean loadMetadata = false;
        boolean run = true;
        while (run) {
            run = false;
            if (loadMetadata) {
                loadMetadataIfNotExist(rpcContext, path, loadMetadataContext, false);
                ufsAccessed = true;
            }
            // We just synced; the new lock pattern should not sync.
            try (LockedInodePath inodePath = mInodeTree.lockInodePath(lockingScheme)) {
                auditContext.setSrcInode(inodePath.getInodeOrNull());
                try {
                    mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
                } catch (AccessControlException e) {
                    auditContext.setAllowed(false);
                    throw e;
                }
                if (!loadMetadata) {
                    Inode inode;
                    boolean isLoaded = true;
                    if (inodePath.fullPathExists()) {
                        inode = inodePath.getInode();
                        if (inode.isDirectory() && context.getOptions().getLoadMetadataType() != LoadMetadataPType.ALWAYS) {
                            InodeDirectory inodeDirectory = inode.asDirectory();
                            isLoaded = inodeDirectory.isDirectChildrenLoaded();
                            if (context.getOptions().getRecursive()) {
                                isLoaded = areDescendantsLoaded(inodeDirectory);
                            }
                            if (isLoaded) {
                                // no need to load again.
                                loadMetadataContext.getOptions().setLoadDescendantType(LoadDescendantPType.NONE);
                            }
                        }
                    } else {
                        checkLoadMetadataOptions(context.getOptions().getLoadMetadataType(), inodePath.getUri());
                    }
                    if (shouldLoadMetadataIfNotExists(inodePath, loadMetadataContext)) {
                        loadMetadata = true;
                        run = true;
                        continue;
                    }
                }
                ensureFullPathAndUpdateCache(inodePath);
                auditContext.setSrcInode(inodePath.getInode());
                MountTable.Resolution resolution;
                if (!context.getOptions().hasLoadMetadataOnly() || !context.getOptions().getLoadMetadataOnly()) {
                    DescendantType descendantTypeForListStatus = (context.getOptions().getRecursive()) ? DescendantType.ALL : DescendantType.ONE;
                    try {
                        resolution = mMountTable.resolve(path);
                    } catch (InvalidPathException e) {
                        throw new FileDoesNotExistException(e.getMessage(), e);
                    }
                    listStatusInternal(context, rpcContext, inodePath, auditContext, descendantTypeForListStatus, resultStream, 0, Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.GET_FILE_INFO));
                    if (!ufsAccessed) {
                        Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.LIST_STATUS).inc();
                    }
                }
                auditContext.setSucceeded(true);
                Metrics.FILE_INFOS_GOT.inc();
            }
        }
    }
}

Example 3 with LoadMetadataContext

use of alluxio.master.file.contexts.LoadMetadataContext in project alluxio by Alluxio.

the class DefaultFileSystemMaster method exists.

@Override
public boolean exists(AlluxioURI path, ExistsContext context) throws AccessControlException, IOException {
    try (RpcContext rpcContext = createRpcContext(context);
        FileSystemMasterAuditContext auditContext = createAuditContext("exists", path, null, null)) {
        syncMetadata(rpcContext, path, context.getOptions().getCommonOptions(), DescendantType.ONE, auditContext, LockedInodePath::getInodeOrNull, (inodePath, permChecker) -> permChecker.checkPermission(Mode.Bits.READ, inodePath), false);
        try (LockedInodePath inodePath = mInodeTree.lockInodePath(createLockingScheme(path, context.getOptions().getCommonOptions(), LockPattern.READ))) {
            LoadMetadataContext lmCtx = LoadMetadataContext.create(LoadMetadataPOptions.newBuilder().setCommonOptions(context.getOptions().getCommonOptions()).setLoadType(context.getOptions().getLoadMetadataType()));
            if (shouldLoadMetadataIfNotExists(inodePath, lmCtx)) {
                checkLoadMetadataOptions(context.getOptions().getLoadMetadataType(), path);
                loadMetadataIfNotExist(rpcContext, path, lmCtx, false);
            }
        } catch (FileDoesNotExistException e) {
            return false;
        }
        try (LockedInodePath inodePath = mInodeTree.lockInodePath(createLockingScheme(path, context.getOptions().getCommonOptions(), LockPattern.READ))) {
            auditContext.setSucceeded(true);
            return inodePath.fullPathExists();
        }
    } catch (InvalidPathException e) {
        return false;
    }
}

Example 4 with LoadMetadataContext

use of alluxio.master.file.contexts.LoadMetadataContext in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getFileIdInternal.

private long getFileIdInternal(AlluxioURI path, boolean checkPermission) throws AccessControlException, UnavailableException {
    try (RpcContext rpcContext = createRpcContext()) {
        /*
      In order to prevent locking twice on RPCs where metadata does _not_ need to be loaded, we use
      a two-step scheme as an optimization to prevent the extra lock. loadMetadataIfNotExists
      requires a lock on the tree to determine if the path should be loaded before executing. To
      prevent the extra lock, we execute the RPC as normal and use a conditional check in the
      main body of the function to determine whether control flow should be shifted out of the
      RPC logic and back to the loadMetadataIfNotExists function.

      If loadMetadataIfNotExists runs, then the next pass into the main logic body should
      continue as normal. This may present a slight decrease in performance for newly-loaded
      metadata, but it is better than affecting the most common case where metadata is not being
      loaded.
       */
        LoadMetadataContext lmCtx = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCreateAncestors(true));
        boolean run = true;
        boolean loadMetadata = false;
        while (run) {
            run = false;
            if (loadMetadata) {
                loadMetadataIfNotExist(rpcContext, path, lmCtx, false);
            }
            try (LockedInodePath inodePath = mInodeTree.lockInodePath(path, LockPattern.READ)) {
                if (checkPermission) {
                    mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
                }
                if (!loadMetadata && shouldLoadMetadataIfNotExists(inodePath, lmCtx)) {
                    loadMetadata = true;
                    run = true;
                    continue;
                }
                mInodeTree.ensureFullInodePath(inodePath);
                return inodePath.getInode().getId();
            } catch (InvalidPathException | FileDoesNotExistException e) {
                return IdUtils.INVALID_FILE_ID;
            }
        }
    } catch (InvalidPathException e) {
        return IdUtils.INVALID_FILE_ID;
    }
    return IdUtils.INVALID_FILE_ID;
}

Example 5 with LoadMetadataContext

use of alluxio.master.file.contexts.LoadMetadataContext in project alluxio by Alluxio.

the class InodeSyncStream method loadMetadataForPath.

private void loadMetadataForPath(LockedInodePath inodePath) throws InvalidPathException, AccessControlException, IOException, FileDoesNotExistException, FileAlreadyCompletedException, InvalidFileSizeException, BlockInfoException {
    UfsStatus status = mStatusCache.fetchStatusIfAbsent(inodePath.getUri(), mMountTable);
    DescendantType descendantType = mDescendantType;
    // do not load the subdirectory
    if (descendantType.equals(DescendantType.ONE) && !inodePath.getUri().equals(mRootScheme.getPath())) {
        descendantType = DescendantType.NONE;
    }
    LoadMetadataContext ctx = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCommonOptions(NO_TTL_OPTION).setCreateAncestors(true).setLoadDescendantType(GrpcUtils.toProto(descendantType))).setUfsStatus(status);
    loadMetadata(inodePath, ctx);
}