Search in sources :

Example 1 with MkdirsOptions

use of alluxio.underfs.options.MkdirsOptions in project alluxio by Alluxio.

the class InodeTree method createPath.

/**
   * Creates a file or directory at path.
   *
   * @param inodePath the path
   * @param options method options
   * @return a {@link CreatePathResult} representing the modified inodes and created inodes during
   *         path creation
   * @throws FileAlreadyExistsException when there is already a file at path if we want to create a
   *         directory there
   * @throws BlockInfoException when blockSizeBytes is invalid
   * @throws InvalidPathException when path is invalid, for example, (1) when there is nonexistent
   *         necessary parent directories and recursive is false, (2) when one of the necessary
   *         parent directories is actually a file
   * @throws IOException if creating the path fails
   * @throws FileDoesNotExistException if the parent of the path does not exist and the recursive
   *         option is false
   */
public CreatePathResult createPath(LockedInodePath inodePath, CreatePathOptions<?> options) throws FileAlreadyExistsException, BlockInfoException, InvalidPathException, IOException, FileDoesNotExistException {
    AlluxioURI path = inodePath.getUri();
    if (path.isRoot()) {
        String errorMessage = ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(path);
        LOG.error(errorMessage);
        throw new FileAlreadyExistsException(errorMessage);
    }
    if (options instanceof CreateFileOptions) {
        CreateFileOptions fileOptions = (CreateFileOptions) options;
        if (fileOptions.getBlockSizeBytes() < 1) {
            throw new BlockInfoException("Invalid block size " + fileOptions.getBlockSizeBytes());
        }
    }
    if (!(inodePath instanceof MutableLockedInodePath)) {
        throw new InvalidPathException(ExceptionMessage.NOT_MUTABLE_INODE_PATH.getMessage(inodePath.getUri()));
    }
    LOG.debug("createPath {}", path);
    TraversalResult traversalResult = traverseToInode(inodePath, LockMode.WRITE_PARENT);
    InodeLockList lockList = traversalResult.getInodeLockList();
    MutableLockedInodePath extensibleInodePath = (MutableLockedInodePath) inodePath;
    String[] pathComponents = extensibleInodePath.getPathComponents();
    String name = path.getName();
    // pathIndex is the index into pathComponents where we start filling in the path from the inode.
    int pathIndex = extensibleInodePath.getInodes().size();
    if (pathIndex < pathComponents.length - 1) {
        // Otherwise we add the remaining path components to the list of components to create.
        if (!options.isRecursive()) {
            final String msg = new StringBuilder().append("File ").append(path).append(" creation failed. Component ").append(pathIndex).append("(").append(pathComponents[pathIndex]).append(") does not exist").toString();
            LOG.error("FileDoesNotExistException: {}", msg);
            throw new FileDoesNotExistException(msg);
        }
    }
    // The ancestor inode (parent or ancestor) of the target path.
    Inode<?> ancestorInode = extensibleInodePath.getAncestorInode();
    if (!ancestorInode.isDirectory()) {
        throw new InvalidPathException("Could not traverse to parent directory of path " + path + ". Component " + pathComponents[pathIndex - 1] + " is not a directory.");
    }
    InodeDirectory currentInodeDirectory = (InodeDirectory) ancestorInode;
    List<Inode<?>> createdInodes = new ArrayList<>();
    List<Inode<?>> modifiedInodes = new ArrayList<>();
    // These are inodes that already exist, that should be journaled as persisted.
    List<Inode<?>> existingNonPersisted = new ArrayList<>();
    // These are inodes to mark as persisted at the end of this method.
    List<Inode<?>> toPersistDirectories = new ArrayList<>();
    if (options.isPersisted()) {
        // Directory persistence will not happen until the end of this method.
        toPersistDirectories.addAll(traversalResult.getNonPersisted());
        existingNonPersisted.addAll(traversalResult.getNonPersisted());
    }
    if (pathIndex < (pathComponents.length - 1) || currentInodeDirectory.getChild(name) == null) {
        // (1) There are components in parent paths that need to be created. Or
        // (2) The last component of the path needs to be created.
        // In these two cases, the last traversed Inode will be modified.
        modifiedInodes.add(currentInodeDirectory);
    }
    // TODO(gpang): We may not have to lock the newly created inodes if the last inode is write
    // locked. This could improve performance. Further investigation is needed.
    // Fill in the ancestor directories that were missing.
    // NOTE, we set the mode of missing ancestor directories to be the default value, rather
    // than inheriting the option of the final file to create, because it may not have
    // "execute" permission.
    CreateDirectoryOptions missingDirOptions = CreateDirectoryOptions.defaults().setMountPoint(false).setPersisted(options.isPersisted()).setOwner(options.getOwner()).setGroup(options.getGroup());
    for (int k = pathIndex; k < (pathComponents.length - 1); k++) {
        InodeDirectory dir = InodeDirectory.create(mDirectoryIdGenerator.getNewDirectoryId(), currentInodeDirectory.getId(), pathComponents[k], missingDirOptions);
        // Lock the newly created inode before subsequent operations, and add it to the lock group.
        lockList.lockWriteAndCheckNameAndParent(dir, currentInodeDirectory, pathComponents[k]);
        dir.setPinned(currentInodeDirectory.isPinned());
        currentInodeDirectory.addChild(dir);
        currentInodeDirectory.setLastModificationTimeMs(options.getOperationTimeMs());
        if (options.isPersisted()) {
            toPersistDirectories.add(dir);
        }
        createdInodes.add(dir);
        mInodes.add(dir);
        currentInodeDirectory = dir;
    }
    // Create the final path component. First we need to make sure that there isn't already a file
    // here with that name. If there is an existing file that is a directory and we're creating a
    // directory, update persistence property of the directories if needed, otherwise, throw
    // FileAlreadyExistsException unless options.allowExists is true.
    Inode<?> lastInode = currentInodeDirectory.getChild(name);
    if (lastInode != null) {
        // Lock the last inode before subsequent operations, and add it to the lock group.
        lockList.lockWriteAndCheckNameAndParent(lastInode, currentInodeDirectory, name);
        if (lastInode.isDirectory() && options instanceof CreateDirectoryOptions && !lastInode.isPersisted() && options.isPersisted()) {
            // The final path component already exists and is not persisted, so it should be added
            // to the non-persisted Inodes of traversalResult.
            existingNonPersisted.add(lastInode);
            toPersistDirectories.add(lastInode);
        } else if (!lastInode.isDirectory() || !(options instanceof CreateDirectoryOptions && ((CreateDirectoryOptions) options).isAllowExists())) {
            String errorMessage = ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(path);
            LOG.error(errorMessage);
            throw new FileAlreadyExistsException(errorMessage);
        }
    } else {
        if (options instanceof CreateDirectoryOptions) {
            CreateDirectoryOptions directoryOptions = (CreateDirectoryOptions) options;
            lastInode = InodeDirectory.create(mDirectoryIdGenerator.getNewDirectoryId(), currentInodeDirectory.getId(), name, directoryOptions);
            // Lock the created inode before subsequent operations, and add it to the lock group.
            lockList.lockWriteAndCheckNameAndParent(lastInode, currentInodeDirectory, name);
            if (directoryOptions.isPersisted()) {
                toPersistDirectories.add(lastInode);
            }
            lastInode.setPinned(currentInodeDirectory.isPinned());
        } else if (options instanceof CreateFileOptions) {
            CreateFileOptions fileOptions = (CreateFileOptions) options;
            lastInode = InodeFile.create(mContainerIdGenerator.getNewContainerId(), currentInodeDirectory.getId(), name, System.currentTimeMillis(), fileOptions);
            // Lock the created inode before subsequent operations, and add it to the lock group.
            lockList.lockWriteAndCheckNameAndParent(lastInode, currentInodeDirectory, name);
            if (currentInodeDirectory.isPinned()) {
                // Update set of pinned file ids.
                mPinnedInodeFileIds.add(lastInode.getId());
            }
            lastInode.setPinned(currentInodeDirectory.isPinned());
        }
        createdInodes.add(lastInode);
        mInodes.add(lastInode);
        currentInodeDirectory.addChild(lastInode);
        currentInodeDirectory.setLastModificationTimeMs(options.getOperationTimeMs());
    }
    // we mark it as persisted.
    for (Inode<?> inode : toPersistDirectories) {
        MountTable.Resolution resolution = mMountTable.resolve(getPath(inode));
        String ufsUri = resolution.getUri().toString();
        UnderFileSystem ufs = resolution.getUfs();
        MkdirsOptions mkdirsOptions = MkdirsOptions.defaults().setCreateParent(false).setOwner(inode.getOwner()).setGroup(inode.getGroup()).setMode(new Mode(inode.getMode()));
        if (ufs.isDirectory(ufsUri) || ufs.mkdirs(ufsUri, mkdirsOptions)) {
            inode.setPersistenceState(PersistenceState.PERSISTED);
        }
    }
    // Extend the inodePath with the created inodes.
    extensibleInodePath.getInodes().addAll(createdInodes);
    LOG.debug("createFile: File Created: {} parent: {}", lastInode, currentInodeDirectory);
    return new CreatePathResult(modifiedInodes, createdInodes, existingNonPersisted);
}
Also used : FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) FileAlreadyExistsException(alluxio.exception.FileAlreadyExistsException) MkdirsOptions(alluxio.underfs.options.MkdirsOptions) CreateDirectoryOptions(alluxio.master.file.options.CreateDirectoryOptions) ArrayList(java.util.ArrayList) InvalidPathException(alluxio.exception.InvalidPathException) CreateFileOptions(alluxio.master.file.options.CreateFileOptions) UnderFileSystem(alluxio.underfs.UnderFileSystem) Mode(alluxio.security.authorization.Mode) BlockInfoException(alluxio.exception.BlockInfoException) AlluxioURI(alluxio.AlluxioURI)

Example 2 with MkdirsOptions

use of alluxio.underfs.options.MkdirsOptions in project alluxio by Alluxio.

the class FileSystemMaster method renameInternal.

/**
   * Implements renaming.
   *
   * @param srcInodePath the path of the rename source
   * @param dstInodePath the path to the rename destination
   * @param replayed whether the operation is a result of replaying the journal
   * @param options method options
   * @throws FileDoesNotExistException if a non-existent file is encountered
   * @throws InvalidPathException if an invalid path is encountered
   * @throws IOException if an I/O error is encountered
   */
private void renameInternal(LockedInodePath srcInodePath, LockedInodePath dstInodePath, boolean replayed, RenameOptions options) throws FileDoesNotExistException, InvalidPathException, IOException {
    // Rename logic:
    // 1. Change the source inode name to the destination name.
    // 2. Insert the source inode into the destination parent.
    // 3. Do UFS operations if necessary.
    // 4. Remove the source inode (reverting the name) from the source parent.
    // 5. Set the last modification times for both source and destination parent inodes.
    Inode<?> srcInode = srcInodePath.getInode();
    AlluxioURI srcPath = srcInodePath.getUri();
    AlluxioURI dstPath = dstInodePath.getUri();
    InodeDirectory srcParentInode = srcInodePath.getParentInodeDirectory();
    InodeDirectory dstParentInode = dstInodePath.getParentInodeDirectory();
    String srcName = srcPath.getName();
    String dstName = dstPath.getName();
    LOG.debug("Renaming {} to {}", srcPath, dstPath);
    // 1. Change the source inode name to the destination name.
    srcInode.setName(dstName);
    srcInode.setParentId(dstParentInode.getId());
    // 2. Insert the source inode into the destination parent.
    if (!dstParentInode.addChild(srcInode)) {
        // On failure, revert changes and throw exception.
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw new InvalidPathException("Destination path: " + dstPath + " already exists.");
    }
    // If the source file is persisted, rename it in the UFS.
    try {
        if (!replayed && srcInode.isPersisted()) {
            MountTable.Resolution resolution = mMountTable.resolve(srcPath);
            String ufsSrcPath = resolution.getUri().toString();
            UnderFileSystem ufs = resolution.getUfs();
            String ufsDstUri = mMountTable.resolve(dstPath).getUri().toString();
            // Create ancestor directories from top to the bottom. We cannot use recursive create
            // parents here because the permission for the ancestors can be different.
            List<Inode<?>> dstInodeList = dstInodePath.getInodeList();
            Stack<Pair<String, MkdirsOptions>> ufsDirsToMakeWithOptions = new Stack<>();
            AlluxioURI curUfsDirPath = new AlluxioURI(ufsDstUri).getParent();
            // The dst inode does not exist yet, so the last inode in the list is the existing parent.
            for (int i = dstInodeList.size() - 1; i >= 0; i--) {
                if (ufs.isDirectory(curUfsDirPath.toString())) {
                    break;
                }
                Inode<?> curInode = dstInodeList.get(i);
                MkdirsOptions mkdirsOptions = MkdirsOptions.defaults().setCreateParent(false).setOwner(curInode.getOwner()).setGroup(curInode.getGroup()).setMode(new Mode(curInode.getMode()));
                ufsDirsToMakeWithOptions.push(new Pair<>(curUfsDirPath.toString(), mkdirsOptions));
                curUfsDirPath = curUfsDirPath.getParent();
            }
            while (!ufsDirsToMakeWithOptions.empty()) {
                Pair<String, MkdirsOptions> ufsDirAndPerm = ufsDirsToMakeWithOptions.pop();
                if (!ufs.mkdirs(ufsDirAndPerm.getFirst(), ufsDirAndPerm.getSecond())) {
                    throw new IOException(ExceptionMessage.FAILED_UFS_CREATE.getMessage(ufsDirAndPerm.getFirst()));
                }
            }
            boolean success;
            if (srcInode.isFile()) {
                success = ufs.renameFile(ufsSrcPath, ufsDstUri);
            } else {
                success = ufs.renameDirectory(ufsSrcPath, ufsDstUri);
            }
            if (!success) {
                throw new IOException(ExceptionMessage.FAILED_UFS_RENAME.getMessage(ufsSrcPath, ufsDstUri));
            }
        }
    } catch (Exception e) {
        // On failure, revert changes and throw exception.
        if (!dstParentInode.removeChild(dstName)) {
            LOG.error("Failed to revert rename changes. Alluxio metadata may be inconsistent.");
        }
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw e;
    }
    // TODO(jiri): A crash between now and the time the rename operation is journaled will result in
    // an inconsistency between Alluxio and UFS.
    // 4. Remove the source inode (reverting the name) from the source parent. The name must be
    // reverted or removeChild will not be able to find the appropriate child entry since it is
    // keyed on the original name.
    srcInode.setName(srcName);
    if (!srcParentInode.removeChild(srcInode)) {
        // This should never happen.
        LOG.error("Failed to rename {} to {} in Alluxio. Alluxio and under storage may be " + "inconsistent.", srcPath, dstPath);
        srcInode.setName(dstName);
        if (!dstParentInode.removeChild(dstName)) {
            LOG.error("Failed to revert changes when renaming {} to {}. Alluxio metadata may be " + "inconsistent.", srcPath, dstPath);
        }
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw new IOException("Failed to remove source path " + srcPath + " from parent");
    }
    srcInode.setName(dstName);
    // 5. Set the last modification times for both source and destination parent inodes.
    // Note this step relies on setLastModificationTimeMs being thread safe to guarantee the
    // correct behavior when multiple files are being renamed within a directory.
    dstParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
    srcParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
    Metrics.PATHS_RENAMED.inc();
}
Also used : MkdirsOptions(alluxio.underfs.options.MkdirsOptions) Mode(alluxio.security.authorization.Mode) IOException(java.io.IOException) MountTable(alluxio.master.file.meta.MountTable) InvalidPathException(alluxio.exception.InvalidPathException) AlluxioException(alluxio.exception.AlluxioException) BlockInfoException(alluxio.exception.BlockInfoException) FileAlreadyExistsException(alluxio.exception.FileAlreadyExistsException) IOException(java.io.IOException) InvalidPathException(alluxio.exception.InvalidPathException) AccessControlException(alluxio.exception.AccessControlException) InvalidFileSizeException(alluxio.exception.InvalidFileSizeException) FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) FileAlreadyCompletedException(alluxio.exception.FileAlreadyCompletedException) DirectoryNotEmptyException(alluxio.exception.DirectoryNotEmptyException) UnexpectedAlluxioException(alluxio.exception.UnexpectedAlluxioException) Stack(java.util.Stack) InodeDirectory(alluxio.master.file.meta.InodeDirectory) Inode(alluxio.master.file.meta.Inode) UnderFileSystem(alluxio.underfs.UnderFileSystem) AlluxioURI(alluxio.AlluxioURI) InodePathPair(alluxio.master.file.meta.InodePathPair) Pair(alluxio.collections.Pair)

Example 3 with MkdirsOptions

use of alluxio.underfs.options.MkdirsOptions in project alluxio by Alluxio.

the class UnderFileSystemUtils method prepareFilePath.

/**
   * Creates parent directories for path with correct permissions if required.
   *
   * @param alluxioPath Alluxio path
   * @param ufsPath path in the under file system
   * @param fs file system master client
   * @param ufs the under file system
   * @throws AlluxioException if an unexpected Alluxio exception is thrown
   * @throws IOException if folder creation fails
   */
public static void prepareFilePath(AlluxioURI alluxioPath, String ufsPath, FileSystem fs, UnderFileSystem ufs) throws AlluxioException, IOException {
    AlluxioURI dstPath = new AlluxioURI(ufsPath);
    String parentPath = dstPath.getParent().getPath();
    // creates the parent folder if it does not exist
    if (!ufs.isDirectory(parentPath)) {
        // Create ancestor directories from top to the bottom. We cannot use recursive create parents
        // here because the permission for the ancestors can be different.
        Stack<Pair<String, MkdirsOptions>> ufsDirsToMakeWithOptions = new Stack<>();
        AlluxioURI curAlluxioPath = alluxioPath.getParent();
        AlluxioURI curUfsPath = dstPath.getParent();
        // Stop at Alluxio mount point because the mapped directory in UFS may not exist.
        while (curUfsPath != null && !ufs.isDirectory(curUfsPath.toString()) && curAlluxioPath != null) {
            URIStatus curDirStatus = fs.getStatus(curAlluxioPath);
            if (curDirStatus.isMountPoint()) {
                throw new IOException(ExceptionMessage.UFS_PATH_DOES_NOT_EXIST.getMessage(curUfsPath));
            }
            ufsDirsToMakeWithOptions.push(new Pair<>(curUfsPath.toString(), MkdirsOptions.defaults().setCreateParent(false).setOwner(curDirStatus.getOwner()).setGroup(curDirStatus.getGroup()).setMode(new Mode((short) curDirStatus.getMode()))));
            curAlluxioPath = curAlluxioPath.getParent();
            curUfsPath = curUfsPath.getParent();
        }
        while (!ufsDirsToMakeWithOptions.empty()) {
            Pair<String, MkdirsOptions> ufsDirAndPerm = ufsDirsToMakeWithOptions.pop();
            // and assume the directory is already prepared, regardless of permission matching.
            if (!ufs.mkdirs(ufsDirAndPerm.getFirst(), ufsDirAndPerm.getSecond()) && !ufs.isDirectory(ufsDirAndPerm.getFirst())) {
                throw new IOException("Failed to create dir: " + ufsDirAndPerm.getFirst());
            }
        }
    }
}
Also used : MkdirsOptions(alluxio.underfs.options.MkdirsOptions) Mode(alluxio.security.authorization.Mode) IOException(java.io.IOException) AlluxioURI(alluxio.AlluxioURI) Stack(java.util.Stack) Pair(alluxio.collections.Pair)

Aggregations

AlluxioURI (alluxio.AlluxioURI)3 Mode (alluxio.security.authorization.Mode)3 MkdirsOptions (alluxio.underfs.options.MkdirsOptions)3 Pair (alluxio.collections.Pair)2 BlockInfoException (alluxio.exception.BlockInfoException)2 FileAlreadyExistsException (alluxio.exception.FileAlreadyExistsException)2 FileDoesNotExistException (alluxio.exception.FileDoesNotExistException)2 InvalidPathException (alluxio.exception.InvalidPathException)2 UnderFileSystem (alluxio.underfs.UnderFileSystem)2 IOException (java.io.IOException)2 Stack (java.util.Stack)2 AccessControlException (alluxio.exception.AccessControlException)1 AlluxioException (alluxio.exception.AlluxioException)1 DirectoryNotEmptyException (alluxio.exception.DirectoryNotEmptyException)1 FileAlreadyCompletedException (alluxio.exception.FileAlreadyCompletedException)1 InvalidFileSizeException (alluxio.exception.InvalidFileSizeException)1 UnexpectedAlluxioException (alluxio.exception.UnexpectedAlluxioException)1 Inode (alluxio.master.file.meta.Inode)1 InodeDirectory (alluxio.master.file.meta.InodeDirectory)1 InodePathPair (alluxio.master.file.meta.InodePathPair)1