Examples with InodeDirectory alluxio.master.file.meta.InodeDirectory used on opensource projects

Example 1 with InodeDirectory

use of alluxio.master.file.meta.InodeDirectory in project alluxio by Alluxio.

the class FileSystemMaster method createDirectoryInternal.

/**
   * Implementation of directory creation for a given path.
   *
   * @param inodePath the path of the directory
   * @param options method options
   * @return an {@link alluxio.master.file.meta.InodeTree.CreatePathResult} representing the
   *         modified inodes and created inodes during path creation
   * @throws InvalidPathException when the path is invalid, please see documentation on {@link
   *         InodeTree#createPath(LockedInodePath, alluxio.master.file.options.CreatePathOptions)}
   *         for more details
   * @throws FileAlreadyExistsException when there is already a file at path
   * @throws IOException if a non-Alluxio related exception occurs
   * @throws AccessControlException if permission checking fails
   */
private InodeTree.CreatePathResult createDirectoryInternal(LockedInodePath inodePath, CreateDirectoryOptions options) throws InvalidPathException, FileAlreadyExistsException, IOException, AccessControlException, FileDoesNotExistException {
    try {
        InodeTree.CreatePathResult createResult = mInodeTree.createPath(inodePath, options);
        InodeDirectory inodeDirectory = (InodeDirectory) inodePath.getInode();
        // If inodeDirectory's ttl not equals Constants.NO_TTL, it should insert into mTtlBuckets
        if (createResult.getCreated().size() > 0) {
            mTtlBuckets.insert(inodeDirectory);
        }
        return createResult;
    } catch (BlockInfoException e) {
        // Since we are creating a directory, the block size is ignored, no such exception should
        // happen.
        Throwables.propagate(e);
    }
    return null;
}

Example 2 with InodeDirectory

use of alluxio.master.file.meta.InodeDirectory in project alluxio by Alluxio.

the class FileSystemMaster method startupCheckConsistency.

/**
   * Checks the consistency of the root in a multi-threaded and incremental fashion. This method
   * will only READ lock the directories and files actively being checked and release them after the
   * check on the file / directory is complete.
   *
   * @return a list of paths in Alluxio which are not consistent with the under storage
   * @throws InterruptedException if the thread is interrupted during execution
   * @throws IOException if an error occurs interacting with the under storage
   */
private List<AlluxioURI> startupCheckConsistency(final ExecutorService service) throws InterruptedException, IOException {
    /** A marker {@link StartupConsistencyChecker}s add to the queue to signal completion */
    final long completionMarker = -1;
    /** A shared queue of directories which have yet to be checked */
    final BlockingQueue<Long> dirsToCheck = new LinkedBlockingQueue<>();
    /**
     * A {@link Callable} which checks the consistency of a directory.
     */
    final class StartupConsistencyChecker implements Callable<List<AlluxioURI>> {

        /** The path to check, guaranteed to be a directory in Alluxio. */
        private final Long mFileId;

        /**
       * Creates a new callable which checks the consistency of a directory.
       * @param fileId the path to check
       */
        private StartupConsistencyChecker(Long fileId) {
            mFileId = fileId;
        }

        /**
       * Checks the consistency of the directory and all immediate children which are files. All
       * immediate children which are directories are added to the shared queue of directories to
       * check. The parent directory is READ locked during the entire call while the children are
       * READ locked only during the consistency check of the children files.
       *
       * @return a list of inconsistent uris
       * @throws IOException if an error occurs interacting with the under storage
       */
        @Override
        public List<AlluxioURI> call() throws IOException {
            List<AlluxioURI> inconsistentUris = new ArrayList<>();
            try (LockedInodePath dir = mInodeTree.lockFullInodePath(mFileId, InodeTree.LockMode.READ)) {
                Inode parentInode = dir.getInode();
                AlluxioURI parentUri = dir.getUri();
                if (!checkConsistencyInternal(parentInode, parentUri)) {
                    inconsistentUris.add(parentUri);
                }
                for (Inode childInode : ((InodeDirectory) parentInode).getChildren()) {
                    try {
                        childInode.lockReadAndCheckParent(parentInode);
                    } catch (InvalidPathException e) {
                        // This should be safe, continue.
                        LOG.debug("Error during startup check consistency, ignoring and continuing.", e);
                        continue;
                    }
                    try {
                        AlluxioURI childUri = parentUri.join(childInode.getName());
                        if (childInode.isDirectory()) {
                            dirsToCheck.add(childInode.getId());
                        } else {
                            if (!checkConsistencyInternal(childInode, childUri)) {
                                inconsistentUris.add(childUri);
                            }
                        }
                    } finally {
                        childInode.unlockRead();
                    }
                }
            } catch (FileDoesNotExistException e) {
                // This should be safe, continue.
                LOG.debug("A file scheduled for consistency check was deleted before the check.");
            } catch (InvalidPathException e) {
                // This should not happen.
                LOG.error("An invalid path was discovered during the consistency check, skipping.", e);
            }
            dirsToCheck.add(completionMarker);
            return inconsistentUris;
        }
    }
    // Add the root to the directories to check.
    dirsToCheck.add(mInodeTree.getRoot().getId());
    List<Future<List<AlluxioURI>>> results = new ArrayList<>();
    // Tracks how many checkers have been started.
    long started = 0;
    // Tracks how many checkers have completed.
    long completed = 0;
    do {
        Long fileId = dirsToCheck.take();
        if (fileId == completionMarker) {
            // A thread signaled completion.
            completed++;
        } else {
            // A new directory needs to be checked.
            StartupConsistencyChecker checker = new StartupConsistencyChecker(fileId);
            results.add(service.submit(checker));
            started++;
        }
    } while (started != completed);
    // Return the total set of inconsistent paths discovered.
    List<AlluxioURI> inconsistentUris = new ArrayList<>();
    for (Future<List<AlluxioURI>> result : results) {
        try {
            inconsistentUris.addAll(result.get());
        } catch (Exception e) {
            // This shouldn't happen, all futures should be complete.
            Throwables.propagate(e);
        }
    }
    service.shutdown();
    return inconsistentUris;
}

Example 3 with InodeDirectory

use of alluxio.master.file.meta.InodeDirectory in project alluxio by Alluxio.

the class FileSystemMaster method loadMetadataAndJournal.

/**
   * Loads metadata for the object identified by the given path from UFS into Alluxio.
   * <p>
   * Writes to the journal.
   *
   * @param inodePath the path for which metadata should be loaded
   * @param options the load metadata options
   * @param journalContext the journal context
   * @throws InvalidPathException if invalid path is encountered
   * @throws FileDoesNotExistException if there is no UFS path
   * @throws BlockInfoException if an invalid block size is encountered
   * @throws FileAlreadyCompletedException if the file is already completed
   * @throws InvalidFileSizeException if invalid file size is encountered
   * @throws AccessControlException if permission checking fails
   * @throws IOException if an I/O error occurs
   */
private void loadMetadataAndJournal(LockedInodePath inodePath, LoadMetadataOptions options, JournalContext journalContext) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException, AccessControlException, IOException {
    AlluxioURI path = inodePath.getUri();
    MountTable.Resolution resolution = mMountTable.resolve(path);
    AlluxioURI ufsUri = resolution.getUri();
    UnderFileSystem ufs = resolution.getUfs();
    try {
        if (options.getUnderFileStatus() == null && !ufs.exists(ufsUri.toString())) {
            // uri does not exist in ufs
            InodeDirectory inode = (InodeDirectory) inodePath.getInode();
            inode.setDirectChildrenLoaded(true);
        }
        boolean isFile;
        if (options.getUnderFileStatus() != null) {
            isFile = options.getUnderFileStatus().isFile();
        } else {
            isFile = ufs.isFile(ufsUri.toString());
        }
        if (isFile) {
            loadFileMetadataAndJournal(inodePath, resolution, options, journalContext);
        } else {
            loadDirectoryMetadataAndJournal(inodePath, options, journalContext);
            InodeDirectory inode = (InodeDirectory) inodePath.getInode();
            if (options.isLoadDirectChildren()) {
                UnderFileStatus[] files = ufs.listStatus(ufsUri.toString());
                for (UnderFileStatus file : files) {
                    if (PathUtils.isTemporaryFileName(file.getName()) || inode.getChild(file.getName()) != null) {
                        continue;
                    }
                    TempInodePathForChild tempInodePath = new TempInodePathForChild(inodePath, file.getName());
                    LoadMetadataOptions loadMetadataOptions = LoadMetadataOptions.defaults().setLoadDirectChildren(false).setCreateAncestors(false).setUnderFileStatus(file);
                    loadMetadataAndJournal(tempInodePath, loadMetadataOptions, journalContext);
                }
                inode.setDirectChildrenLoaded(true);
            }
        }
    } catch (IOException e) {
        LOG.error(ExceptionUtils.getStackTrace(e));
        throw e;
    }
}

Example 4 with InodeDirectory

use of alluxio.master.file.meta.InodeDirectory in project alluxio by Alluxio.

the class FileSystemMaster method freeAndJournal.

/**
   * Implements free operation.
   * <p>
   * This may write to the journal as free operation may change the pinned state of inodes.
   *
   * @param inodePath inode of the path to free
   * @param options options to free
   * @param journalContext the journal context
   * @throws FileDoesNotExistException if the file does not exist
   * @throws AccessControlException if permission checking fails
   * @throws InvalidPathException if the given path is invalid
   * @throws UnexpectedAlluxioException if the file or directory can not be freed
   */
private void freeAndJournal(LockedInodePath inodePath, FreeOptions options, JournalContext journalContext) throws FileDoesNotExistException, UnexpectedAlluxioException, AccessControlException, InvalidPathException {
    Inode<?> inode = inodePath.getInode();
    if (inode.isDirectory() && !options.isRecursive() && ((InodeDirectory) inode).getNumberOfChildren() > 0) {
        // inode is nonempty, and we don't free a nonempty directory unless recursive is true
        throw new UnexpectedAlluxioException(ExceptionMessage.CANNOT_FREE_NON_EMPTY_DIR.getMessage(mInodeTree.getPath(inode)));
    }
    long opTimeMs = System.currentTimeMillis();
    List<Inode<?>> freeInodes = new ArrayList<>();
    freeInodes.add(inode);
    try (InodeLockList lockList = mInodeTree.lockDescendants(inodePath, InodeTree.LockMode.WRITE)) {
        freeInodes.addAll(lockList.getInodes());
        TempInodePathForDescendant tempInodePath = new TempInodePathForDescendant(inodePath);
        // We go through each inode.
        for (int i = freeInodes.size() - 1; i >= 0; i--) {
            Inode<?> freeInode = freeInodes.get(i);
            if (freeInode.isFile()) {
                if (freeInode.getPersistenceState() != PersistenceState.PERSISTED) {
                    throw new UnexpectedAlluxioException(ExceptionMessage.CANNOT_FREE_NON_PERSISTED_FILE.getMessage(mInodeTree.getPath(freeInode)));
                }
                if (freeInode.isPinned()) {
                    if (!options.isForced()) {
                        throw new UnexpectedAlluxioException(ExceptionMessage.CANNOT_FREE_PINNED_FILE.getMessage(mInodeTree.getPath(freeInode)));
                    }
                    // the path to inode for getPath should already be locked.
                    tempInodePath.setDescendant(freeInode, mInodeTree.getPath(freeInode));
                    SetAttributeOptions setAttributeOptions = SetAttributeOptions.defaults().setRecursive(false).setPinned(false);
                    setAttributeInternal(tempInodePath, false, opTimeMs, setAttributeOptions);
                    journalSetAttribute(tempInodePath, opTimeMs, setAttributeOptions, journalContext);
                }
                // Remove corresponding blocks from workers.
                mBlockMaster.removeBlocks(((InodeFile) freeInode).getBlockIds(), false);
            }
        }
    }
    Metrics.FILES_FREED.inc(freeInodes.size());
}

Example 5 with InodeDirectory

use of alluxio.master.file.meta.InodeDirectory in project alluxio by Alluxio.

the class FileSystemMaster method renameInternal.

/**
   * Implements renaming.
   *
   * @param srcInodePath the path of the rename source
   * @param dstInodePath the path to the rename destination
   * @param replayed whether the operation is a result of replaying the journal
   * @param options method options
   * @throws FileDoesNotExistException if a non-existent file is encountered
   * @throws InvalidPathException if an invalid path is encountered
   * @throws IOException if an I/O error is encountered
   */
private void renameInternal(LockedInodePath srcInodePath, LockedInodePath dstInodePath, boolean replayed, RenameOptions options) throws FileDoesNotExistException, InvalidPathException, IOException {
    // Rename logic:
    // 1. Change the source inode name to the destination name.
    // 2. Insert the source inode into the destination parent.
    // 3. Do UFS operations if necessary.
    // 4. Remove the source inode (reverting the name) from the source parent.
    // 5. Set the last modification times for both source and destination parent inodes.
    Inode<?> srcInode = srcInodePath.getInode();
    AlluxioURI srcPath = srcInodePath.getUri();
    AlluxioURI dstPath = dstInodePath.getUri();
    InodeDirectory srcParentInode = srcInodePath.getParentInodeDirectory();
    InodeDirectory dstParentInode = dstInodePath.getParentInodeDirectory();
    String srcName = srcPath.getName();
    String dstName = dstPath.getName();
    LOG.debug("Renaming {} to {}", srcPath, dstPath);
    // 1. Change the source inode name to the destination name.
    srcInode.setName(dstName);
    srcInode.setParentId(dstParentInode.getId());
    // 2. Insert the source inode into the destination parent.
    if (!dstParentInode.addChild(srcInode)) {
        // On failure, revert changes and throw exception.
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw new InvalidPathException("Destination path: " + dstPath + " already exists.");
    }
    // If the source file is persisted, rename it in the UFS.
    try {
        if (!replayed && srcInode.isPersisted()) {
            MountTable.Resolution resolution = mMountTable.resolve(srcPath);
            String ufsSrcPath = resolution.getUri().toString();
            UnderFileSystem ufs = resolution.getUfs();
            String ufsDstUri = mMountTable.resolve(dstPath).getUri().toString();
            // Create ancestor directories from top to the bottom. We cannot use recursive create
            // parents here because the permission for the ancestors can be different.
            List<Inode<?>> dstInodeList = dstInodePath.getInodeList();
            Stack<Pair<String, MkdirsOptions>> ufsDirsToMakeWithOptions = new Stack<>();
            AlluxioURI curUfsDirPath = new AlluxioURI(ufsDstUri).getParent();
            // The dst inode does not exist yet, so the last inode in the list is the existing parent.
            for (int i = dstInodeList.size() - 1; i >= 0; i--) {
                if (ufs.isDirectory(curUfsDirPath.toString())) {
                    break;
                }
                Inode<?> curInode = dstInodeList.get(i);
                MkdirsOptions mkdirsOptions = MkdirsOptions.defaults().setCreateParent(false).setOwner(curInode.getOwner()).setGroup(curInode.getGroup()).setMode(new Mode(curInode.getMode()));
                ufsDirsToMakeWithOptions.push(new Pair<>(curUfsDirPath.toString(), mkdirsOptions));
                curUfsDirPath = curUfsDirPath.getParent();
            }
            while (!ufsDirsToMakeWithOptions.empty()) {
                Pair<String, MkdirsOptions> ufsDirAndPerm = ufsDirsToMakeWithOptions.pop();
                if (!ufs.mkdirs(ufsDirAndPerm.getFirst(), ufsDirAndPerm.getSecond())) {
                    throw new IOException(ExceptionMessage.FAILED_UFS_CREATE.getMessage(ufsDirAndPerm.getFirst()));
                }
            }
            boolean success;
            if (srcInode.isFile()) {
                success = ufs.renameFile(ufsSrcPath, ufsDstUri);
            } else {
                success = ufs.renameDirectory(ufsSrcPath, ufsDstUri);
            }
            if (!success) {
                throw new IOException(ExceptionMessage.FAILED_UFS_RENAME.getMessage(ufsSrcPath, ufsDstUri));
            }
        }
    } catch (Exception e) {
        // On failure, revert changes and throw exception.
        if (!dstParentInode.removeChild(dstName)) {
            LOG.error("Failed to revert rename changes. Alluxio metadata may be inconsistent.");
        }
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw e;
    }
    // TODO(jiri): A crash between now and the time the rename operation is journaled will result in
    // an inconsistency between Alluxio and UFS.
    // 4. Remove the source inode (reverting the name) from the source parent. The name must be
    // reverted or removeChild will not be able to find the appropriate child entry since it is
    // keyed on the original name.
    srcInode.setName(srcName);
    if (!srcParentInode.removeChild(srcInode)) {
        // This should never happen.
        LOG.error("Failed to rename {} to {} in Alluxio. Alluxio and under storage may be " + "inconsistent.", srcPath, dstPath);
        srcInode.setName(dstName);
        if (!dstParentInode.removeChild(dstName)) {
            LOG.error("Failed to revert changes when renaming {} to {}. Alluxio metadata may be " + "inconsistent.", srcPath, dstPath);
        }
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw new IOException("Failed to remove source path " + srcPath + " from parent");
    }
    srcInode.setName(dstName);
    // 5. Set the last modification times for both source and destination parent inodes.
    // Note this step relies on setLastModificationTimeMs being thread safe to guarantee the
    // correct behavior when multiple files are being renamed within a directory.
    dstParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
    srcParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
    Metrics.PATHS_RENAMED.inc();
}