Search in sources :

Example 6 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class FileSystemMaster method completeFile.

/**
   * Completes a file. After a file is completed, it cannot be written to.
   * <p>
   * This operation requires users to have {@link Mode.Bits#WRITE} permission on the path.
   *
   * @param path the file path to complete
   * @param options the method options
   * @throws BlockInfoException if a block information exception is encountered
   * @throws FileDoesNotExistException if the file does not exist
   * @throws InvalidPathException if an invalid path is encountered
   * @throws InvalidFileSizeException if an invalid file size is encountered
   * @throws FileAlreadyCompletedException if the file is already completed
   * @throws AccessControlException if permission checking fails
   */
public void completeFile(AlluxioURI path, CompleteFileOptions options) throws BlockInfoException, FileDoesNotExistException, InvalidPathException, InvalidFileSizeException, FileAlreadyCompletedException, AccessControlException {
    Metrics.COMPLETE_FILE_OPS.inc();
    try (JournalContext journalContext = createJournalContext();
        LockedInodePath inodePath = mInodeTree.lockFullInodePath(path, InodeTree.LockMode.WRITE)) {
        mPermissionChecker.checkPermission(Mode.Bits.WRITE, inodePath);
        // Even readonly mount points should be able to complete a file, for UFS reads in CACHE mode.
        completeFileAndJournal(inodePath, options, journalContext);
    }
}
Also used : LockedInodePath(alluxio.master.file.meta.LockedInodePath)

Example 7 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class FileSystemMaster method setAttributeFromEntry.

/**
   * @param entry the entry to use
   * @throws FileDoesNotExistException if the file does not exist
   * @throws InvalidPathException if the file path corresponding to the file id is invalid
   * @throws AccessControlException if failed to set permission
   */
private void setAttributeFromEntry(SetAttributeEntry entry) throws FileDoesNotExistException, InvalidPathException, AccessControlException {
    SetAttributeOptions options = SetAttributeOptions.defaults();
    if (entry.hasPinned()) {
        options.setPinned(entry.getPinned());
    }
    if (entry.hasTtl()) {
        options.setTtl(entry.getTtl());
        options.setTtlAction(ProtobufUtils.fromProtobuf(entry.getTtlAction()));
    }
    if (entry.hasPersisted()) {
        options.setPersisted(entry.getPersisted());
    }
    if (entry.hasOwner()) {
        options.setOwner(entry.getOwner());
    }
    if (entry.hasGroup()) {
        options.setGroup(entry.getGroup());
    }
    if (entry.hasPermission()) {
        options.setMode((short) entry.getPermission());
    }
    try (LockedInodePath inodePath = mInodeTree.lockFullInodePath(entry.getId(), InodeTree.LockMode.WRITE)) {
        setAttributeInternal(inodePath, true, entry.getOpTimeMs(), options);
    // Intentionally not journaling the persisted inodes from setAttributeInternal
    }
}
Also used : SetAttributeOptions(alluxio.master.file.options.SetAttributeOptions) LockedInodePath(alluxio.master.file.meta.LockedInodePath)

Example 8 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class FileSystemMaster method startupCheckConsistency.

/**
   * Checks the consistency of the root in a multi-threaded and incremental fashion. This method
   * will only READ lock the directories and files actively being checked and release them after the
   * check on the file / directory is complete.
   *
   * @return a list of paths in Alluxio which are not consistent with the under storage
   * @throws InterruptedException if the thread is interrupted during execution
   * @throws IOException if an error occurs interacting with the under storage
   */
private List<AlluxioURI> startupCheckConsistency(final ExecutorService service) throws InterruptedException, IOException {
    /** A marker {@link StartupConsistencyChecker}s add to the queue to signal completion */
    final long completionMarker = -1;
    /** A shared queue of directories which have yet to be checked */
    final BlockingQueue<Long> dirsToCheck = new LinkedBlockingQueue<>();
    /**
     * A {@link Callable} which checks the consistency of a directory.
     */
    final class StartupConsistencyChecker implements Callable<List<AlluxioURI>> {

        /** The path to check, guaranteed to be a directory in Alluxio. */
        private final Long mFileId;

        /**
       * Creates a new callable which checks the consistency of a directory.
       * @param fileId the path to check
       */
        private StartupConsistencyChecker(Long fileId) {
            mFileId = fileId;
        }

        /**
       * Checks the consistency of the directory and all immediate children which are files. All
       * immediate children which are directories are added to the shared queue of directories to
       * check. The parent directory is READ locked during the entire call while the children are
       * READ locked only during the consistency check of the children files.
       *
       * @return a list of inconsistent uris
       * @throws IOException if an error occurs interacting with the under storage
       */
        @Override
        public List<AlluxioURI> call() throws IOException {
            List<AlluxioURI> inconsistentUris = new ArrayList<>();
            try (LockedInodePath dir = mInodeTree.lockFullInodePath(mFileId, InodeTree.LockMode.READ)) {
                Inode parentInode = dir.getInode();
                AlluxioURI parentUri = dir.getUri();
                if (!checkConsistencyInternal(parentInode, parentUri)) {
                    inconsistentUris.add(parentUri);
                }
                for (Inode childInode : ((InodeDirectory) parentInode).getChildren()) {
                    try {
                        childInode.lockReadAndCheckParent(parentInode);
                    } catch (InvalidPathException e) {
                        // This should be safe, continue.
                        LOG.debug("Error during startup check consistency, ignoring and continuing.", e);
                        continue;
                    }
                    try {
                        AlluxioURI childUri = parentUri.join(childInode.getName());
                        if (childInode.isDirectory()) {
                            dirsToCheck.add(childInode.getId());
                        } else {
                            if (!checkConsistencyInternal(childInode, childUri)) {
                                inconsistentUris.add(childUri);
                            }
                        }
                    } finally {
                        childInode.unlockRead();
                    }
                }
            } catch (FileDoesNotExistException e) {
                // This should be safe, continue.
                LOG.debug("A file scheduled for consistency check was deleted before the check.");
            } catch (InvalidPathException e) {
                // This should not happen.
                LOG.error("An invalid path was discovered during the consistency check, skipping.", e);
            }
            dirsToCheck.add(completionMarker);
            return inconsistentUris;
        }
    }
    // Add the root to the directories to check.
    dirsToCheck.add(mInodeTree.getRoot().getId());
    List<Future<List<AlluxioURI>>> results = new ArrayList<>();
    // Tracks how many checkers have been started.
    long started = 0;
    // Tracks how many checkers have completed.
    long completed = 0;
    do {
        Long fileId = dirsToCheck.take();
        if (fileId == completionMarker) {
            // A thread signaled completion.
            completed++;
        } else {
            // A new directory needs to be checked.
            StartupConsistencyChecker checker = new StartupConsistencyChecker(fileId);
            results.add(service.submit(checker));
            started++;
        }
    } while (started != completed);
    // Return the total set of inconsistent paths discovered.
    List<AlluxioURI> inconsistentUris = new ArrayList<>();
    for (Future<List<AlluxioURI>> result : results) {
        try {
            inconsistentUris.addAll(result.get());
        } catch (Exception e) {
            // This shouldn't happen, all futures should be complete.
            Throwables.propagate(e);
        }
    }
    service.shutdown();
    return inconsistentUris;
}
Also used : FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) ArrayList(java.util.ArrayList) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) InvalidPathException(alluxio.exception.InvalidPathException) Callable(java.util.concurrent.Callable) AlluxioException(alluxio.exception.AlluxioException) BlockInfoException(alluxio.exception.BlockInfoException) FileAlreadyExistsException(alluxio.exception.FileAlreadyExistsException) IOException(java.io.IOException) InvalidPathException(alluxio.exception.InvalidPathException) AccessControlException(alluxio.exception.AccessControlException) InvalidFileSizeException(alluxio.exception.InvalidFileSizeException) FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) FileAlreadyCompletedException(alluxio.exception.FileAlreadyCompletedException) DirectoryNotEmptyException(alluxio.exception.DirectoryNotEmptyException) UnexpectedAlluxioException(alluxio.exception.UnexpectedAlluxioException) LockedInodePath(alluxio.master.file.meta.LockedInodePath) InodeDirectory(alluxio.master.file.meta.InodeDirectory) Inode(alluxio.master.file.meta.Inode) Future(java.util.concurrent.Future) ArrayList(java.util.ArrayList) InodeLockList(alluxio.master.file.meta.InodeLockList) List(java.util.List) TtlBucketList(alluxio.master.file.meta.TtlBucketList) PrefixList(alluxio.collections.PrefixList) AlluxioURI(alluxio.AlluxioURI)

Example 9 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class FileSystemMaster method resetFile.

/**
   * Resets a file. It first free the whole file, and then reinitializes it.
   *
   * @param fileId the id of the file
   * @throws FileDoesNotExistException if the file does not exist
   * @throws AccessControlException if permission checking fails
   * @throws InvalidPathException if the path is invalid for the id of the file
   * @throws UnexpectedAlluxioException if the file or directory can not be freed
   */
// Currently used by Lineage Master
// TODO(binfan): Add permission checking for internal APIs
public void resetFile(long fileId) throws UnexpectedAlluxioException, FileDoesNotExistException, InvalidPathException, AccessControlException {
    // TODO(yupeng) check the file is not persisted
    try (JournalContext journalContext = createJournalContext();
        LockedInodePath inodePath = mInodeTree.lockFullInodePath(fileId, InodeTree.LockMode.WRITE)) {
        // free the file first
        InodeFile inodeFile = inodePath.getInodeFile();
        freeAndJournal(inodePath, FreeOptions.defaults().setForced(true), journalContext);
        inodeFile.reset();
    }
}
Also used : LockedInodePath(alluxio.master.file.meta.LockedInodePath) InodeFile(alluxio.master.file.meta.InodeFile)

Example 10 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class FileSystemMaster method createDirectory.

/**
   * Creates a directory for a given path.
   * <p>
   * This operation requires the client user to have
   * {@link Mode.Bits#WRITE} permission on the parent of the path.
   *
   * @param path the path of the directory
   * @param options method options
   * @return the id of the created directory
   * @throws InvalidPathException when the path is invalid, please see documentation on {@link
   *         InodeTree#createPath(LockedInodePath, alluxio.master.file.options.CreatePathOptions)}
   *         for more details
   * @throws FileAlreadyExistsException when there is already a file at path
   * @throws IOException if a non-Alluxio related exception occurs
   * @throws AccessControlException if permission checking fails
   * @throws FileDoesNotExistException if the parent of the path does not exist and the recursive
   *         option is false
   */
public long createDirectory(AlluxioURI path, CreateDirectoryOptions options) throws InvalidPathException, FileAlreadyExistsException, IOException, AccessControlException, FileDoesNotExistException {
    LOG.debug("createDirectory {} ", path);
    Metrics.CREATE_DIRECTORIES_OPS.inc();
    try (JournalContext journalContext = createJournalContext();
        LockedInodePath inodePath = mInodeTree.lockInodePath(path, InodeTree.LockMode.WRITE)) {
        mPermissionChecker.checkParentPermission(Mode.Bits.WRITE, inodePath);
        mMountTable.checkUnderWritableMountPoint(path);
        createDirectoryAndJournal(inodePath, options, journalContext);
        return inodePath.getInode().getId();
    }
}
Also used : LockedInodePath(alluxio.master.file.meta.LockedInodePath)

Aggregations

LockedInodePath (alluxio.master.file.meta.LockedInodePath)79 AlluxioURI (alluxio.AlluxioURI)27 AccessControlException (alluxio.exception.AccessControlException)24 FileDoesNotExistException (alluxio.exception.FileDoesNotExistException)23 InvalidPathException (alluxio.exception.InvalidPathException)18 LockingScheme (alluxio.master.file.meta.LockingScheme)16 Inode (alluxio.master.file.meta.Inode)15 ArrayList (java.util.ArrayList)13 Test (org.junit.Test)11 Mode (alluxio.security.authorization.Mode)10 LockedInodePathList (alluxio.master.file.meta.LockedInodePathList)9 BlockInfoException (alluxio.exception.BlockInfoException)8 FileAlreadyExistsException (alluxio.exception.FileAlreadyExistsException)8 IOException (java.io.IOException)8 FileAlreadyCompletedException (alluxio.exception.FileAlreadyCompletedException)7 InvalidFileSizeException (alluxio.exception.InvalidFileSizeException)7 LoadMetadataContext (alluxio.master.file.contexts.LoadMetadataContext)7 InodeFile (alluxio.master.file.meta.InodeFile)7 MountTable (alluxio.master.file.meta.MountTable)7 UnexpectedAlluxioException (alluxio.exception.UnexpectedAlluxioException)6