Examples with AlluxioURI alluxio.AlluxioURI used on opensource projects

Example 46 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class FileSystemMaster method mountInternal.

/**
   * Updates the mount table with the specified mount point. The mount options may be updated during
   * this method.
   *
   * @param inodePath the Alluxio mount point
   * @param ufsPath the UFS endpoint to mount
   * @param replayed whether the operation is a result of replaying the journal
   * @param options the mount options (may be updated)
   * @throws FileAlreadyExistsException if the mount point already exists
   * @throws InvalidPathException if an invalid path is encountered
   * @throws IOException if an I/O exception occurs
   */
private void mountInternal(LockedInodePath inodePath, AlluxioURI ufsPath, boolean replayed, MountOptions options) throws FileAlreadyExistsException, InvalidPathException, IOException {
    AlluxioURI alluxioPath = inodePath.getUri();
    if (!replayed) {
        // Check that the ufsPath exists and is a directory
        UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsPath.toString());
        ufs.setProperties(options.getProperties());
        if (!ufs.isDirectory(ufsPath.toString())) {
            throw new IOException(ExceptionMessage.UFS_PATH_DOES_NOT_EXIST.getMessage(ufsPath.getPath()));
        }
        // Check that the alluxioPath we're creating doesn't shadow a path in the default UFS
        String defaultUfsPath = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
        UnderFileSystem defaultUfs = UnderFileSystem.Factory.get(defaultUfsPath);
        String shadowPath = PathUtils.concatPath(defaultUfsPath, alluxioPath.getPath());
        if (defaultUfs.exists(shadowPath)) {
            throw new IOException(ExceptionMessage.MOUNT_PATH_SHADOWS_DEFAULT_UFS.getMessage(alluxioPath));
        }
        // Configure the ufs properties, and update the mount options with the configured properties.
        ufs.configureProperties();
        options.setProperties(ufs.getProperties());
    }
    // Add the mount point. This will only succeed if we are not mounting a prefix of an existing
    // mount and no existing mount is a prefix of this mount.
    mMountTable.add(alluxioPath, ufsPath, options);
}

Example 47 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class FileSystemMaster method propagatePersistedInternal.

/**
   * Propagates the persisted status to all parents of the given inode in the same mount partition.
   *
   * @param inodePath the inode to start the propagation at
   * @param replayed whether the invocation is a result of replaying the journal
   * @return list of inodes which were marked as persisted
   * @throws FileDoesNotExistException if a non-existent file is encountered
   */
private List<Inode<?>> propagatePersistedInternal(LockedInodePath inodePath, boolean replayed) throws FileDoesNotExistException {
    Inode<?> inode = inodePath.getInode();
    if (!inode.isPersisted()) {
        return Collections.emptyList();
    }
    List<Inode<?>> inodes = inodePath.getInodeList();
    // Traverse the inodes from target inode to the root.
    Collections.reverse(inodes);
    // Skip the first, to not examine the target inode itself.
    inodes = inodes.subList(1, inodes.size());
    List<Inode<?>> persistedInodes = new ArrayList<>();
    for (Inode<?> handle : inodes) {
        // the path is already locked.
        AlluxioURI path = mInodeTree.getPath(handle);
        if (mMountTable.isMountPoint(path)) {
            // Stop propagating the persisted status at mount points.
            break;
        }
        if (handle.isPersisted()) {
            // Stop if a persisted directory is encountered.
            break;
        }
        handle.setPersistenceState(PersistenceState.PERSISTED);
        if (!replayed) {
            persistedInodes.add(inode);
        }
    }
    return persistedInodes;
}

Example 48 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class FileSystemMaster method startupCheckConsistency.

/**
   * Checks the consistency of the root in a multi-threaded and incremental fashion. This method
   * will only READ lock the directories and files actively being checked and release them after the
   * check on the file / directory is complete.
   *
   * @return a list of paths in Alluxio which are not consistent with the under storage
   * @throws InterruptedException if the thread is interrupted during execution
   * @throws IOException if an error occurs interacting with the under storage
   */
private List<AlluxioURI> startupCheckConsistency(final ExecutorService service) throws InterruptedException, IOException {
    /** A marker {@link StartupConsistencyChecker}s add to the queue to signal completion */
    final long completionMarker = -1;
    /** A shared queue of directories which have yet to be checked */
    final BlockingQueue<Long> dirsToCheck = new LinkedBlockingQueue<>();
    /**
     * A {@link Callable} which checks the consistency of a directory.
     */
    final class StartupConsistencyChecker implements Callable<List<AlluxioURI>> {

        /** The path to check, guaranteed to be a directory in Alluxio. */
        private final Long mFileId;

        /**
       * Creates a new callable which checks the consistency of a directory.
       * @param fileId the path to check
       */
        private StartupConsistencyChecker(Long fileId) {
            mFileId = fileId;
        }

        /**
       * Checks the consistency of the directory and all immediate children which are files. All
       * immediate children which are directories are added to the shared queue of directories to
       * check. The parent directory is READ locked during the entire call while the children are
       * READ locked only during the consistency check of the children files.
       *
       * @return a list of inconsistent uris
       * @throws IOException if an error occurs interacting with the under storage
       */
        @Override
        public List<AlluxioURI> call() throws IOException {
            List<AlluxioURI> inconsistentUris = new ArrayList<>();
            try (LockedInodePath dir = mInodeTree.lockFullInodePath(mFileId, InodeTree.LockMode.READ)) {
                Inode parentInode = dir.getInode();
                AlluxioURI parentUri = dir.getUri();
                if (!checkConsistencyInternal(parentInode, parentUri)) {
                    inconsistentUris.add(parentUri);
                }
                for (Inode childInode : ((InodeDirectory) parentInode).getChildren()) {
                    try {
                        childInode.lockReadAndCheckParent(parentInode);
                    } catch (InvalidPathException e) {
                        // This should be safe, continue.
                        LOG.debug("Error during startup check consistency, ignoring and continuing.", e);
                        continue;
                    }
                    try {
                        AlluxioURI childUri = parentUri.join(childInode.getName());
                        if (childInode.isDirectory()) {
                            dirsToCheck.add(childInode.getId());
                        } else {
                            if (!checkConsistencyInternal(childInode, childUri)) {
                                inconsistentUris.add(childUri);
                            }
                        }
                    } finally {
                        childInode.unlockRead();
                    }
                }
            } catch (FileDoesNotExistException e) {
                // This should be safe, continue.
                LOG.debug("A file scheduled for consistency check was deleted before the check.");
            } catch (InvalidPathException e) {
                // This should not happen.
                LOG.error("An invalid path was discovered during the consistency check, skipping.", e);
            }
            dirsToCheck.add(completionMarker);
            return inconsistentUris;
        }
    }
    // Add the root to the directories to check.
    dirsToCheck.add(mInodeTree.getRoot().getId());
    List<Future<List<AlluxioURI>>> results = new ArrayList<>();
    // Tracks how many checkers have been started.
    long started = 0;
    // Tracks how many checkers have completed.
    long completed = 0;
    do {
        Long fileId = dirsToCheck.take();
        if (fileId == completionMarker) {
            // A thread signaled completion.
            completed++;
        } else {
            // A new directory needs to be checked.
            StartupConsistencyChecker checker = new StartupConsistencyChecker(fileId);
            results.add(service.submit(checker));
            started++;
        }
    } while (started != completed);
    // Return the total set of inconsistent paths discovered.
    List<AlluxioURI> inconsistentUris = new ArrayList<>();
    for (Future<List<AlluxioURI>> result : results) {
        try {
            inconsistentUris.addAll(result.get());
        } catch (Exception e) {
            // This shouldn't happen, all futures should be complete.
            Throwables.propagate(e);
        }
    }
    service.shutdown();
    return inconsistentUris;
}

Example 49 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class S3UnderFileSystemFactory method create.

@Override
public UnderFileSystem create(String path, Object unusedConf) {
    Preconditions.checkNotNull(path);
    if (checkAWSCredentials()) {
        try {
            return S3UnderFileSystem.createInstance(new AlluxioURI(path));
        } catch (ServiceException e) {
            throw Throwables.propagate(e);
        }
    }
    String err = "AWS Credentials not available, cannot create S3 Under File System.";
    throw Throwables.propagate(new IOException(err));
}

Example 50 with AlluxioURI

use of alluxio.AlluxioURI in project alluxio by Alluxio.

the class SwiftUnderFileSystemFactory method create.

@Override
public UnderFileSystem create(String path, Object unusedConf) {
    Preconditions.checkNotNull(path);
    if (addAndCheckSwiftCredentials()) {
        try {
            return new SwiftUnderFileSystem(new AlluxioURI(path));
        } catch (Exception e) {
            LOG.error("Failed to create SwiftUnderFileSystem.", e);
            throw Throwables.propagate(e);
        }
    }
    String err = "Swift Credentials not available, cannot create Swift Under File System.";
    LOG.error(err);
    throw Throwables.propagate(new IOException(err));
}