Examples with Inode alluxio.master.file.meta.Inode used on opensource projects

Example 31 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class UfsSyncChecker method checkDirectory.

/**
 * Check if immediate children of directory are in sync with UFS.
 *
 * @param inode read-locked directory to check
 * @param alluxioUri path of directory to to check
 */
@SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
public void checkDirectory(InodeDirectory inode, AlluxioURI alluxioUri) throws FileDoesNotExistException, InvalidPathException, IOException {
    Preconditions.checkArgument(inode.isPersisted());
    UfsStatus[] ufsChildren = getChildrenInUFS(alluxioUri);
    // Filter out temporary files
    ufsChildren = Arrays.stream(ufsChildren).filter(ufsStatus -> !PathUtils.isTemporaryFileName(ufsStatus.getName())).toArray(UfsStatus[]::new);
    Arrays.sort(ufsChildren, Comparator.comparing(UfsStatus::getName));
    Inode[] alluxioChildren = Iterables.toArray(mInodeStore.getChildren(inode), Inode.class);
    Arrays.sort(alluxioChildren);
    int ufsPos = 0;
    for (Inode alluxioInode : alluxioChildren) {
        if (ufsPos >= ufsChildren.length) {
            break;
        }
        String ufsName = ufsChildren[ufsPos].getName();
        if (ufsName.endsWith(AlluxioURI.SEPARATOR)) {
            ufsName = ufsName.substring(0, ufsName.length() - 1);
        }
        if (ufsName.equals(alluxioInode.getName())) {
            ufsPos++;
        }
    }
    if (ufsPos == ufsChildren.length) {
        // Directory is in sync
        mSyncedDirectories.put(alluxioUri, inode);
    } else {
        // Invalidate ancestor directories if not a mount point
        AlluxioURI currentPath = alluxioUri;
        while (currentPath.getParent() != null && !mMountTable.isMountPoint(currentPath) && mSyncedDirectories.containsKey(currentPath.getParent())) {
            mSyncedDirectories.remove(currentPath.getParent());
            currentPath = currentPath.getParent();
        }
        LOG.debug("Ufs file {} does not match any file in Alluxio", ufsChildren[ufsPos]);
    }
}

Example 32 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method listStatus.

@Override
public void listStatus(AlluxioURI path, ListStatusContext context, ResultStream<FileInfo> resultStream) throws AccessControlException, FileDoesNotExistException, InvalidPathException, IOException {
    Metrics.GET_FILE_INFO_OPS.inc();
    LockingScheme lockingScheme = new LockingScheme(path, LockPattern.READ, false);
    boolean ufsAccessed = false;
    try (RpcContext rpcContext = createRpcContext(context);
        FileSystemMasterAuditContext auditContext = createAuditContext("listStatus", path, null, null)) {
        DescendantType descendantType = context.getOptions().getRecursive() ? DescendantType.ALL : DescendantType.ONE;
        if (!syncMetadata(rpcContext, path, context.getOptions().getCommonOptions(), descendantType, auditContext, LockedInodePath::getInodeOrNull, (inodePath, permChecker) -> permChecker.checkPermission(Mode.Bits.READ, inodePath), false).equals(NOT_NEEDED)) {
            // If synced, do not load metadata.
            context.getOptions().setLoadMetadataType(LoadMetadataPType.NEVER);
            ufsAccessed = true;
        }
        /*
      See the comments in #getFileIdInternal for an explanation on why the loop here is required.
       */
        DescendantType loadDescendantType;
        if (context.getOptions().getLoadMetadataType() == LoadMetadataPType.NEVER) {
            loadDescendantType = DescendantType.NONE;
        } else if (context.getOptions().getRecursive()) {
            loadDescendantType = DescendantType.ALL;
        } else {
            loadDescendantType = DescendantType.ONE;
        }
        // load metadata for 1 level of descendants, or all descendants if recursive
        LoadMetadataContext loadMetadataContext = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCreateAncestors(true).setLoadType(context.getOptions().getLoadMetadataType()).setLoadDescendantType(GrpcUtils.toProto(loadDescendantType)).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction())));
        boolean loadMetadata = false;
        boolean run = true;
        while (run) {
            run = false;
            if (loadMetadata) {
                loadMetadataIfNotExist(rpcContext, path, loadMetadataContext, false);
                ufsAccessed = true;
            }
            // We just synced; the new lock pattern should not sync.
            try (LockedInodePath inodePath = mInodeTree.lockInodePath(lockingScheme)) {
                auditContext.setSrcInode(inodePath.getInodeOrNull());
                try {
                    mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
                } catch (AccessControlException e) {
                    auditContext.setAllowed(false);
                    throw e;
                }
                if (!loadMetadata) {
                    Inode inode;
                    boolean isLoaded = true;
                    if (inodePath.fullPathExists()) {
                        inode = inodePath.getInode();
                        if (inode.isDirectory() && context.getOptions().getLoadMetadataType() != LoadMetadataPType.ALWAYS) {
                            InodeDirectory inodeDirectory = inode.asDirectory();
                            isLoaded = inodeDirectory.isDirectChildrenLoaded();
                            if (context.getOptions().getRecursive()) {
                                isLoaded = areDescendantsLoaded(inodeDirectory);
                            }
                            if (isLoaded) {
                                // no need to load again.
                                loadMetadataContext.getOptions().setLoadDescendantType(LoadDescendantPType.NONE);
                            }
                        }
                    } else {
                        checkLoadMetadataOptions(context.getOptions().getLoadMetadataType(), inodePath.getUri());
                    }
                    if (shouldLoadMetadataIfNotExists(inodePath, loadMetadataContext)) {
                        loadMetadata = true;
                        run = true;
                        continue;
                    }
                }
                ensureFullPathAndUpdateCache(inodePath);
                auditContext.setSrcInode(inodePath.getInode());
                MountTable.Resolution resolution;
                if (!context.getOptions().hasLoadMetadataOnly() || !context.getOptions().getLoadMetadataOnly()) {
                    DescendantType descendantTypeForListStatus = (context.getOptions().getRecursive()) ? DescendantType.ALL : DescendantType.ONE;
                    try {
                        resolution = mMountTable.resolve(path);
                    } catch (InvalidPathException e) {
                        throw new FileDoesNotExistException(e.getMessage(), e);
                    }
                    listStatusInternal(context, rpcContext, inodePath, auditContext, descendantTypeForListStatus, resultStream, 0, Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.GET_FILE_INFO));
                    if (!ufsAccessed) {
                        Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.LIST_STATUS).inc();
                    }
                }
                auditContext.setSucceeded(true);
                Metrics.FILE_INFOS_GOT.inc();
            }
        }
    }
}

Example 33 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method checkConsistencyInternal.

/**
 * Checks if a path is consistent between Alluxio and the underlying storage.
 * <p>
 * A path without a backing under storage is always consistent.
 * <p>
 * A not persisted path is considered consistent if:
 * 1. It does not shadow an object in the underlying storage.
 * <p>
 * A persisted path is considered consistent if:
 * 1. An equivalent object exists for its under storage path.
 * 2. The metadata of the Alluxio and under storage object are equal.
 *
 * @param inodePath the path to check. This must exist and be read-locked
 * @return true if the path is consistent, false otherwise
 */
private boolean checkConsistencyInternal(LockedInodePath inodePath) throws InvalidPathException, IOException {
    Inode inode;
    try {
        inode = inodePath.getInode();
    } catch (FileDoesNotExistException e) {
        // already checked existence when creating the inodePath
        throw new RuntimeException(e);
    }
    MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        String ufsPath = resolution.getUri().getPath();
        if (ufs == null) {
            return true;
        }
        if (!inode.isPersisted()) {
            return !ufs.exists(ufsPath);
        }
        UfsStatus ufsStatus;
        try {
            ufsStatus = ufs.getStatus(ufsPath);
        } catch (FileNotFoundException e) {
            return !inode.isPersisted();
        }
        // TODO(calvin): Evaluate which other metadata fields should be validated.
        if (inode.isDirectory()) {
            return ufsStatus.isDirectory();
        } else {
            String ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
            return ufsStatus.isFile() && (ufsFingerprint.equals(inode.asFile().getUfsFingerprint())) && ufsStatus instanceof UfsFileStatus && ((UfsFileStatus) ufsStatus).getContentLength() == inode.asFile().getLength();
        }
    }
}

Example 34 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method deleteInternal.

/**
 * Implements file deletion.
 * <p>
 * This method does not delete blocks. Instead, it returns deleted inodes so that their blocks can
 * be deleted after the inode deletion journal entry has been written. We cannot delete blocks
 * earlier because the inode deletion may fail, leaving us with inode containing deleted blocks.
 *
 * This method is used at:
 * (1) delete()
 * (2) unmount()
 * (3) metadata sync (when a file/dir has been removed in UFS)
 * Permission check should be skipped in (2) and (3).
 *
 * @param rpcContext the rpc context
 * @param inodePath the file {@link LockedInodePath}
 * @param deleteContext the method optitions
 * @param bypassPermCheck whether the permission check has been done before entering this call
 */
@VisibleForTesting
public void deleteInternal(RpcContext rpcContext, LockedInodePath inodePath, DeleteContext deleteContext, boolean bypassPermCheck) throws FileDoesNotExistException, IOException, DirectoryNotEmptyException, InvalidPathException {
    Preconditions.checkState(inodePath.getLockPattern() == LockPattern.WRITE_EDGE);
    // journaled will result in an inconsistency between Alluxio and UFS.
    if (!inodePath.fullPathExists()) {
        return;
    }
    long opTimeMs = System.currentTimeMillis();
    Inode inode = inodePath.getInode();
    if (inode == null) {
        return;
    }
    boolean recursive = deleteContext.getOptions().getRecursive();
    if (inode.isDirectory() && !recursive && mInodeStore.hasChildren(inode.asDirectory())) {
        // true
        throw new DirectoryNotEmptyException(ExceptionMessage.DELETE_NONEMPTY_DIRECTORY_NONRECURSIVE, inode.getName());
    }
    if (mInodeTree.isRootId(inode.getId())) {
        // The root cannot be deleted.
        throw new InvalidPathException(ExceptionMessage.DELETE_ROOT_DIRECTORY.getMessage());
    }
    // Inodes for which deletion will be attempted
    List<Pair<AlluxioURI, LockedInodePath>> inodesToDelete;
    if (inode.isDirectory()) {
        inodesToDelete = new ArrayList<>((int) inode.asDirectory().getChildCount());
    } else {
        inodesToDelete = new ArrayList<>(1);
    }
    // Add root of sub-tree to delete
    inodesToDelete.add(new Pair<>(inodePath.getUri(), inodePath));
    // Inodes that are not safe for recursive deletes
    // Issues#15266: This can be replaced by a Trie<Long> using prefix matching
    Set<Long> unsafeInodes = new HashSet<>();
    // Unsafe parents due to containing a child which cannot be deleted
    // are initially contained in a separate set, allowing their children
    // to be deleted for which the user has permissions
    Set<Long> unsafeParentInodes = new HashSet<>();
    // Alluxio URIs (and the reason for failure) which could not be deleted
    List<Pair<String, String>> failedUris = new ArrayList<>();
    try (LockedInodePathList descendants = mInodeTree.getDescendants(inodePath)) {
        // Therefore, we first see a parent, then all its children.
        for (LockedInodePath childPath : descendants) {
            if (bypassPermCheck) {
                inodesToDelete.add(new Pair<>(mInodeTree.getPath(childPath.getInode()), childPath));
            } else {
                try {
                    // Because we first see the parent then all its children
                    if (unsafeInodes.contains(childPath.getAncestorInode().getId())) {
                        // We still need to add this child to the unsafe set because we are going to
                        // walk over this child's children.
                        unsafeInodes.add(childPath.getInode().getId());
                        continue;
                    }
                    mPermissionChecker.checkPermission(Mode.Bits.WRITE, childPath);
                    inodesToDelete.add(new Pair<>(mInodeTree.getPath(childPath.getInode()), childPath));
                } catch (AccessControlException e) {
                    // If we do not have permission to delete the inode, then add to unsafe set
                    Inode inodeToDelete = childPath.getInode();
                    unsafeInodes.add(inodeToDelete.getId());
                    // Propagate 'unsafe-ness' to parent as one of its descendants can't be deleted
                    unsafeParentInodes.add(inodeToDelete.getParentId());
                    // All this node's children will be skipped in the failure message
                    failedUris.add(new Pair<>(childPath.toString(), e.getMessage()));
                }
            }
        }
        unsafeInodes.addAll(unsafeParentInodes);
        // Prepare to delete persisted inodes
        UfsDeleter ufsDeleter = NoopUfsDeleter.INSTANCE;
        if (!deleteContext.getOptions().getAlluxioOnly()) {
            ufsDeleter = new SafeUfsDeleter(mMountTable, mInodeStore, inodesToDelete, deleteContext.getOptions().build());
        }
        // file, we deal with the checkpoints and blocks as well.
        for (int i = inodesToDelete.size() - 1; i >= 0; i--) {
            rpcContext.throwIfCancelled();
            Pair<AlluxioURI, LockedInodePath> inodePairToDelete = inodesToDelete.get(i);
            AlluxioURI alluxioUriToDelete = inodePairToDelete.getFirst();
            Inode inodeToDelete = inodePairToDelete.getSecond().getInode();
            String failureReason = null;
            if (unsafeInodes.contains(inodeToDelete.getId())) {
                failureReason = ExceptionMessage.DELETE_FAILED_DIR_NONEMPTY.getMessage();
            } else if (inodeToDelete.isPersisted()) {
                // TODO(calvin): Add tests (ALLUXIO-1831)
                if (mMountTable.isMountPoint(alluxioUriToDelete)) {
                    mMountTable.delete(rpcContext, alluxioUriToDelete, true);
                } else {
                    if (!deleteContext.getOptions().getAlluxioOnly()) {
                        try {
                            checkUfsMode(alluxioUriToDelete, OperationType.WRITE);
                            // Attempt to delete node if all children were deleted successfully
                            ufsDeleter.delete(alluxioUriToDelete, inodeToDelete);
                        } catch (AccessControlException | IOException e) {
                            // In case ufs is not writable, we will still attempt to delete other entries
                            // if any as they may be from a different mount point
                            LOG.warn("Failed to delete {}: {}", alluxioUriToDelete, e.toString());
                            failureReason = e.getMessage();
                        }
                    }
                }
            }
            if (failureReason == null) {
                if (inodeToDelete.isFile()) {
                    long fileId = inodeToDelete.getId();
                    // Remove the file from the set of files to persist.
                    mPersistRequests.remove(fileId);
                    // Cancel any ongoing jobs.
                    PersistJob job = mPersistJobs.get(fileId);
                    if (job != null) {
                        job.setCancelState(PersistJob.CancelState.TO_BE_CANCELED);
                    }
                }
            } else {
                unsafeInodes.add(inodeToDelete.getId());
                // Propagate 'unsafe-ness' to parent as one of its descendants can't be deleted
                unsafeInodes.add(inodeToDelete.getParentId());
                failedUris.add(new Pair<>(alluxioUriToDelete.toString(), failureReason));
                // Something went wrong with this path so it cannot be removed normally
                // Remove the path from further processing
                inodesToDelete.set(i, null);
            }
        }
        if (mSyncManager.isSyncPoint(inodePath.getUri())) {
            mSyncManager.stopSyncAndJournal(RpcContext.NOOP, inodePath.getUri());
        }
        // Delete Inodes from children to parents
        for (int i = inodesToDelete.size() - 1; i >= 0; i--) {
            Pair<AlluxioURI, LockedInodePath> delInodePair = inodesToDelete.get(i);
            // The entry is null because an error is met from the pre-processing
            if (delInodePair == null) {
                continue;
            }
            LockedInodePath tempInodePath = delInodePair.getSecond();
            MountTable.Resolution resolution = mMountTable.resolve(tempInodePath.getUri());
            mInodeTree.deleteInode(rpcContext, tempInodePath, opTimeMs);
            if (deleteContext.getOptions().getAlluxioOnly()) {
                Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.DELETE_FILE).inc();
            }
        }
        if (!failedUris.isEmpty()) {
            throw new FailedPreconditionException(buildDeleteFailureMessage(failedUris));
        }
    }
    Metrics.PATHS_DELETED.inc(inodesToDelete.size());
}

Example 35 with Inode

use of alluxio.master.file.meta.Inode in project alluxio by Alluxio.

the class DefaultFileSystemMaster method shouldLoadMetadataIfNotExists.

boolean shouldLoadMetadataIfNotExists(LockedInodePath inodePath, LoadMetadataContext context) {
    boolean inodeExists = inodePath.fullPathExists();
    boolean loadDirectChildren = false;
    if (inodeExists) {
        try {
            Inode inode = inodePath.getInode();
            loadDirectChildren = inode.isDirectory() && (context.getOptions().getLoadDescendantType() != LoadDescendantPType.NONE);
        } catch (FileDoesNotExistException e) {
            // This should never happen.
            throw new RuntimeException(e);
        }
    }
    return !inodeExists || loadDirectChildren;
}