Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 71 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class JournalWriterTest method rotateLogOnSyncException.

@Test
public void rotateLogOnSyncException() throws Exception {
    // Setup so that we can trigger an IOException when sync is called on the underlying stream.
    JournalWriter mockJournalWriter = PowerMockito.mock(JournalWriter.class);
    FSDataOutputStream mockOutStream = mock(FSDataOutputStream.class);
    UnderFileSystem mockUfs = mock(UnderFileSystem.class);
    doReturn(mockOutStream).when(mockUfs).create(eq(mJournal.getCurrentLogFilePath()), any(CreateOptions.class));
    EntryOutputStream entryOutStream = new EntryOutputStream(mockUfs, mJournal.getCurrentLogFilePath(), mJournal.getJournalFormatter(), mockJournalWriter);
    entryOutStream.writeEntry(JournalEntry.newBuilder().build());
    doThrow(new IOException("sync failed")).when(mockOutStream).sync();
    try {
        entryOutStream.flush();
        Assert.fail("Should have thrown an exception");
    } catch (IOException ignored) {
    // expected
    }
    // Undo the earlier mocking so that the writeEntry doesn't fail.
    doNothing().when(mockOutStream).sync();
    // The rotation happens the next time an entry is written.
    entryOutStream.writeEntry(JournalEntry.newBuilder().build());
    verify(mockJournalWriter).completeCurrentLog();
}

Example 72 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method loadMetadataAndJournal.

/**
   * Loads metadata for the object identified by the given path from UFS into Alluxio.
   * <p>
   * Writes to the journal.
   *
   * @param inodePath the path for which metadata should be loaded
   * @param options the load metadata options
   * @param journalContext the journal context
   * @throws InvalidPathException if invalid path is encountered
   * @throws FileDoesNotExistException if there is no UFS path
   * @throws BlockInfoException if an invalid block size is encountered
   * @throws FileAlreadyCompletedException if the file is already completed
   * @throws InvalidFileSizeException if invalid file size is encountered
   * @throws AccessControlException if permission checking fails
   * @throws IOException if an I/O error occurs
   */
private void loadMetadataAndJournal(LockedInodePath inodePath, LoadMetadataOptions options, JournalContext journalContext) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException, AccessControlException, IOException {
    AlluxioURI path = inodePath.getUri();
    MountTable.Resolution resolution = mMountTable.resolve(path);
    AlluxioURI ufsUri = resolution.getUri();
    UnderFileSystem ufs = resolution.getUfs();
    try {
        if (options.getUnderFileStatus() == null && !ufs.exists(ufsUri.toString())) {
            // uri does not exist in ufs
            InodeDirectory inode = (InodeDirectory) inodePath.getInode();
            inode.setDirectChildrenLoaded(true);
        }
        boolean isFile;
        if (options.getUnderFileStatus() != null) {
            isFile = options.getUnderFileStatus().isFile();
        } else {
            isFile = ufs.isFile(ufsUri.toString());
        }
        if (isFile) {
            loadFileMetadataAndJournal(inodePath, resolution, options, journalContext);
        } else {
            loadDirectoryMetadataAndJournal(inodePath, options, journalContext);
            InodeDirectory inode = (InodeDirectory) inodePath.getInode();
            if (options.isLoadDirectChildren()) {
                UnderFileStatus[] files = ufs.listStatus(ufsUri.toString());
                for (UnderFileStatus file : files) {
                    if (PathUtils.isTemporaryFileName(file.getName()) || inode.getChild(file.getName()) != null) {
                        continue;
                    }
                    TempInodePathForChild tempInodePath = new TempInodePathForChild(inodePath, file.getName());
                    LoadMetadataOptions loadMetadataOptions = LoadMetadataOptions.defaults().setLoadDirectChildren(false).setCreateAncestors(false).setUnderFileStatus(file);
                    loadMetadataAndJournal(tempInodePath, loadMetadataOptions, journalContext);
                }
                inode.setDirectChildrenLoaded(true);
            }
        }
    } catch (IOException e) {
        LOG.error(ExceptionUtils.getStackTrace(e));
        throw e;
    }
}

Example 73 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method loadDirectoryMetadataAndJournal.

/**
   * Loads metadata for the directory identified by the given path from UFS into Alluxio. This does
   * not actually require looking at the UFS path.
   * It is a no-op if the directory exists and is persisted.
   *
   * @param inodePath the path for which metadata should be loaded
   * @param options the load metadata options
   * @param journalContext the journal context
   * @throws InvalidPathException if invalid path is encountered
   * @throws IOException if an I/O error occurs
   * @throws AccessControlException if permission checking fails
   * @throws FileDoesNotExistException if the path does not exist
   */
private void loadDirectoryMetadataAndJournal(LockedInodePath inodePath, LoadMetadataOptions options, JournalContext journalContext) throws FileDoesNotExistException, InvalidPathException, AccessControlException, IOException {
    if (inodePath.fullPathExists()) {
        if (inodePath.getInode().isPersisted()) {
            return;
        }
    }
    CreateDirectoryOptions createDirectoryOptions = CreateDirectoryOptions.defaults().setMountPoint(mMountTable.isMountPoint(inodePath.getUri())).setPersisted(true).setRecursive(options.isCreateAncestors()).setMetadataLoad(true).setAllowExists(true);
    MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
    AlluxioURI ufsUri = resolution.getUri();
    UnderFileSystem ufs = resolution.getUfs();
    String ufsOwner = ufs.getOwner(ufsUri.toString());
    String ufsGroup = ufs.getGroup(ufsUri.toString());
    short ufsMode = ufs.getMode(ufsUri.toString());
    Mode mode = new Mode(ufsMode);
    if (resolution.getShared()) {
        mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits()));
    }
    createDirectoryOptions = createDirectoryOptions.setOwner(ufsOwner).setGroup(ufsGroup).setMode(mode);
    try {
        createDirectoryAndJournal(inodePath, createDirectoryOptions, journalContext);
    } catch (FileAlreadyExistsException e) {
        // This should not happen.
        throw new RuntimeException(e);
    }
}

Example 74 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method renameInternal.

/**
   * Implements renaming.
   *
   * @param srcInodePath the path of the rename source
   * @param dstInodePath the path to the rename destination
   * @param replayed whether the operation is a result of replaying the journal
   * @param options method options
   * @throws FileDoesNotExistException if a non-existent file is encountered
   * @throws InvalidPathException if an invalid path is encountered
   * @throws IOException if an I/O error is encountered
   */
private void renameInternal(LockedInodePath srcInodePath, LockedInodePath dstInodePath, boolean replayed, RenameOptions options) throws FileDoesNotExistException, InvalidPathException, IOException {
    // Rename logic:
    // 1. Change the source inode name to the destination name.
    // 2. Insert the source inode into the destination parent.
    // 3. Do UFS operations if necessary.
    // 4. Remove the source inode (reverting the name) from the source parent.
    // 5. Set the last modification times for both source and destination parent inodes.
    Inode<?> srcInode = srcInodePath.getInode();
    AlluxioURI srcPath = srcInodePath.getUri();
    AlluxioURI dstPath = dstInodePath.getUri();
    InodeDirectory srcParentInode = srcInodePath.getParentInodeDirectory();
    InodeDirectory dstParentInode = dstInodePath.getParentInodeDirectory();
    String srcName = srcPath.getName();
    String dstName = dstPath.getName();
    LOG.debug("Renaming {} to {}", srcPath, dstPath);
    // 1. Change the source inode name to the destination name.
    srcInode.setName(dstName);
    srcInode.setParentId(dstParentInode.getId());
    // 2. Insert the source inode into the destination parent.
    if (!dstParentInode.addChild(srcInode)) {
        // On failure, revert changes and throw exception.
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw new InvalidPathException("Destination path: " + dstPath + " already exists.");
    }
    // If the source file is persisted, rename it in the UFS.
    try {
        if (!replayed && srcInode.isPersisted()) {
            MountTable.Resolution resolution = mMountTable.resolve(srcPath);
            String ufsSrcPath = resolution.getUri().toString();
            UnderFileSystem ufs = resolution.getUfs();
            String ufsDstUri = mMountTable.resolve(dstPath).getUri().toString();
            // Create ancestor directories from top to the bottom. We cannot use recursive create
            // parents here because the permission for the ancestors can be different.
            List<Inode<?>> dstInodeList = dstInodePath.getInodeList();
            Stack<Pair<String, MkdirsOptions>> ufsDirsToMakeWithOptions = new Stack<>();
            AlluxioURI curUfsDirPath = new AlluxioURI(ufsDstUri).getParent();
            // The dst inode does not exist yet, so the last inode in the list is the existing parent.
            for (int i = dstInodeList.size() - 1; i >= 0; i--) {
                if (ufs.isDirectory(curUfsDirPath.toString())) {
                    break;
                }
                Inode<?> curInode = dstInodeList.get(i);
                MkdirsOptions mkdirsOptions = MkdirsOptions.defaults().setCreateParent(false).setOwner(curInode.getOwner()).setGroup(curInode.getGroup()).setMode(new Mode(curInode.getMode()));
                ufsDirsToMakeWithOptions.push(new Pair<>(curUfsDirPath.toString(), mkdirsOptions));
                curUfsDirPath = curUfsDirPath.getParent();
            }
            while (!ufsDirsToMakeWithOptions.empty()) {
                Pair<String, MkdirsOptions> ufsDirAndPerm = ufsDirsToMakeWithOptions.pop();
                if (!ufs.mkdirs(ufsDirAndPerm.getFirst(), ufsDirAndPerm.getSecond())) {
                    throw new IOException(ExceptionMessage.FAILED_UFS_CREATE.getMessage(ufsDirAndPerm.getFirst()));
                }
            }
            boolean success;
            if (srcInode.isFile()) {
                success = ufs.renameFile(ufsSrcPath, ufsDstUri);
            } else {
                success = ufs.renameDirectory(ufsSrcPath, ufsDstUri);
            }
            if (!success) {
                throw new IOException(ExceptionMessage.FAILED_UFS_RENAME.getMessage(ufsSrcPath, ufsDstUri));
            }
        }
    } catch (Exception e) {
        // On failure, revert changes and throw exception.
        if (!dstParentInode.removeChild(dstName)) {
            LOG.error("Failed to revert rename changes. Alluxio metadata may be inconsistent.");
        }
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw e;
    }
    // TODO(jiri): A crash between now and the time the rename operation is journaled will result in
    // an inconsistency between Alluxio and UFS.
    // 4. Remove the source inode (reverting the name) from the source parent. The name must be
    // reverted or removeChild will not be able to find the appropriate child entry since it is
    // keyed on the original name.
    srcInode.setName(srcName);
    if (!srcParentInode.removeChild(srcInode)) {
        // This should never happen.
        LOG.error("Failed to rename {} to {} in Alluxio. Alluxio and under storage may be " + "inconsistent.", srcPath, dstPath);
        srcInode.setName(dstName);
        if (!dstParentInode.removeChild(dstName)) {
            LOG.error("Failed to revert changes when renaming {} to {}. Alluxio metadata may be " + "inconsistent.", srcPath, dstPath);
        }
        srcInode.setName(srcName);
        srcInode.setParentId(srcParentInode.getId());
        throw new IOException("Failed to remove source path " + srcPath + " from parent");
    }
    srcInode.setName(dstName);
    // 5. Set the last modification times for both source and destination parent inodes.
    // Note this step relies on setLastModificationTimeMs being thread safe to guarantee the
    // correct behavior when multiple files are being renamed within a directory.
    dstParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
    srcParentInode.setLastModificationTimeMs(options.getOperationTimeMs());
    Metrics.PATHS_RENAMED.inc();
}

Example 75 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method deleteInternal.

/**
   * Implements file deletion.
   *
   * @param inodePath the file {@link LockedInodePath}
   * @param replayed whether the operation is a result of replaying the journal
   * @param opTimeMs the time of the operation
   * @param deleteOptions the method optitions
   * @throws FileDoesNotExistException if a non-existent file is encountered
   * @throws IOException if an I/O error is encountered
   * @throws InvalidPathException if the specified path is the root
   * @throws DirectoryNotEmptyException if recursive is false and the file is a nonempty directory
   */
private void deleteInternal(LockedInodePath inodePath, boolean replayed, long opTimeMs, DeleteOptions deleteOptions) throws FileDoesNotExistException, IOException, DirectoryNotEmptyException, InvalidPathException {
    // journaled will result in an inconsistency between Alluxio and UFS.
    if (!inodePath.fullPathExists()) {
        return;
    }
    Inode<?> inode = inodePath.getInode();
    if (inode == null) {
        return;
    }
    boolean recursive = deleteOptions.isRecursive();
    boolean alluxioOnly = deleteOptions.isAlluxioOnly();
    if (inode.isDirectory() && !recursive && ((InodeDirectory) inode).getNumberOfChildren() > 0) {
        // true
        throw new DirectoryNotEmptyException(ExceptionMessage.DELETE_NONEMPTY_DIRECTORY_NONRECURSIVE, inode.getName());
    }
    if (mInodeTree.isRootId(inode.getId())) {
        // The root cannot be deleted.
        throw new InvalidPathException(ExceptionMessage.DELETE_ROOT_DIRECTORY.getMessage());
    }
    List<Inode<?>> delInodes = new ArrayList<>();
    delInodes.add(inode);
    try (InodeLockList lockList = mInodeTree.lockDescendants(inodePath, InodeTree.LockMode.WRITE)) {
        delInodes.addAll(lockList.getInodes());
        TempInodePathForDescendant tempInodePath = new TempInodePathForDescendant(inodePath);
        // file, we deal with the checkpoints and blocks as well.
        for (int i = delInodes.size() - 1; i >= 0; i--) {
            Inode<?> delInode = delInodes.get(i);
            // the path to delInode for getPath should already be locked.
            AlluxioURI alluxioUriToDel = mInodeTree.getPath(delInode);
            tempInodePath.setDescendant(delInode, alluxioUriToDel);
            // Currently, it will result in an inconsistency between Alluxio and UFS.
            if (!replayed && delInode.isPersisted()) {
                try {
                    // TODO(calvin): Add tests (ALLUXIO-1831)
                    if (mMountTable.isMountPoint(alluxioUriToDel)) {
                        unmountInternal(alluxioUriToDel);
                    } else {
                        // Delete the file in the under file system.
                        MountTable.Resolution resolution = mMountTable.resolve(alluxioUriToDel);
                        String ufsUri = resolution.getUri().toString();
                        UnderFileSystem ufs = resolution.getUfs();
                        boolean failedToDelete = false;
                        if (!alluxioOnly) {
                            if (delInode.isFile()) {
                                if (!ufs.deleteFile(ufsUri)) {
                                    failedToDelete = ufs.isFile(ufsUri);
                                    if (!failedToDelete) {
                                        LOG.warn("The file to delete does not exist in ufs: {}", ufsUri);
                                    }
                                }
                            } else {
                                if (!ufs.deleteDirectory(ufsUri, alluxio.underfs.options.DeleteOptions.defaults().setRecursive(true))) {
                                    failedToDelete = ufs.isDirectory(ufsUri);
                                    if (!failedToDelete) {
                                        LOG.warn("The directory to delete does not exist in ufs: {}", ufsUri);
                                    }
                                }
                            }
                            if (failedToDelete) {
                                LOG.error("Failed to delete {} from the under filesystem", ufsUri);
                                throw new IOException(ExceptionMessage.DELETE_FAILED_UFS.getMessage(ufsUri));
                            }
                        }
                    }
                } catch (InvalidPathException e) {
                    LOG.warn(e.getMessage());
                }
            }
            if (delInode.isFile()) {
                // Remove corresponding blocks from workers and delete metadata in master.
                mBlockMaster.removeBlocks(((InodeFile) delInode).getBlockIds(), true);
            }
            mInodeTree.deleteInode(tempInodePath, opTimeMs);
        }
    }
    Metrics.PATHS_DELETED.inc(delInodes.size());
}