Examples with UfsStatus alluxio.underfs.UfsStatus used on opensource projects

Example 1 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class DefaultFileSystemMaster method checkConsistencyRecursive.

private void checkConsistencyRecursive(LockedInodePath inodePath, List<AlluxioURI> inconsistentUris, boolean assertInconsistent, boolean metadataSynced) throws IOException, FileDoesNotExistException {
    Inode inode = inodePath.getInode();
    try {
        if (assertInconsistent || !checkConsistencyInternal(inodePath)) {
            inconsistentUris.add(inodePath.getUri());
            // If a dir in Alluxio is inconsistent with underlying storage,
            // we can assert the children is inconsistent.
            // If a file is inconsistent, please ignore this parameter cause it has no child node.
            assertInconsistent = true;
        }
        if (inode.isDirectory()) {
            InodeDirectory inodeDir = inode.asDirectory();
            Iterable<? extends Inode> children = mInodeStore.getChildren(inodeDir);
            for (Inode child : children) {
                try (LockedInodePath childPath = inodePath.lockChild(child, LockPattern.READ)) {
                    checkConsistencyRecursive(childPath, inconsistentUris, assertInconsistent, metadataSynced);
                }
            }
            // if the metadata has already been synced, then we could skip it.
            if (metadataSynced) {
                return;
            }
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            UfsStatus[] statuses;
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                String ufsPath = resolution.getUri().getPath();
                statuses = ufs.listStatus(ufsPath);
            }
            if (statuses != null) {
                HashSet<String> alluxioFileNames = Streams.stream(children).map(Inode::getName).collect(Collectors.toCollection(HashSet::new));
                Arrays.stream(statuses).forEach(status -> {
                    if (!alluxioFileNames.contains(status.getName())) {
                        inconsistentUris.add(inodePath.getUri().join(status.getName()));
                    }
                });
            }
        }
    } catch (InvalidPathException e) {
        LOG.debug("Path \"{}\" is invalid, has been ignored.", PathUtils.concatPath(inodePath.getUri().getPath()));
    }
}

Example 2 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class DefaultFileSystemMaster method completeFileInternal.

/**
 * Completes a file. After a file is completed, it cannot be written to.
 *
 * @param rpcContext the rpc context
 * @param inodePath the {@link LockedInodePath} to complete
 * @param context the method context
 */
void completeFileInternal(RpcContext rpcContext, LockedInodePath inodePath, CompleteFileContext context) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException, UnavailableException {
    Inode inode = inodePath.getInode();
    if (!inode.isFile()) {
        throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(inodePath.getUri()));
    }
    InodeFile fileInode = inode.asFile();
    List<Long> blockIdList = fileInode.getBlockIds();
    List<BlockInfo> blockInfoList = mBlockMaster.getBlockInfoList(blockIdList);
    if (!fileInode.isPersisted() && blockInfoList.size() != blockIdList.size()) {
        throw new BlockInfoException("Cannot complete a file without all the blocks committed");
    }
    // Iterate over all file blocks committed to Alluxio, computing the length and verify that all
    // the blocks (except the last one) is the same size as the file block size.
    long inAlluxioLength = 0;
    long fileBlockSize = fileInode.getBlockSizeBytes();
    for (int i = 0; i < blockInfoList.size(); i++) {
        BlockInfo blockInfo = blockInfoList.get(i);
        inAlluxioLength += blockInfo.getLength();
        if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) {
            throw new BlockInfoException("Block index " + i + " has a block size smaller than the file block size (" + fileInode.getBlockSizeBytes() + ")");
        }
    }
    // If the file is persisted, its length is determined by UFS. Otherwise, its length is
    // determined by its size in Alluxio.
    long length = fileInode.isPersisted() ? context.getOptions().getUfsLength() : inAlluxioLength;
    String ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
    if (fileInode.isPersisted()) {
        UfsStatus ufsStatus = context.getUfsStatus();
        // Retrieve the UFS fingerprint for this file.
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        AlluxioURI resolvedUri = resolution.getUri();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            if (ufsStatus == null) {
                ufsFingerprint = ufs.getFingerprint(resolvedUri.toString());
            } else {
                ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
            }
        }
    }
    completeFileInternal(rpcContext, inodePath, length, context.getOperationTimeMs(), ufsFingerprint);
}

Example 3 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class DefaultFileSystemMaster method createDirectoryInternal.

/**
 * Implementation of directory creation for a given path.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path of the directory
 * @param context method context
 * @return a list of created inodes
 */
List<Inode> createDirectoryInternal(RpcContext rpcContext, LockedInodePath inodePath, CreateDirectoryContext context) throws InvalidPathException, FileAlreadyExistsException, IOException, FileDoesNotExistException {
    Preconditions.checkState(inodePath.getLockPattern() == LockPattern.WRITE_EDGE);
    try {
        List<Inode> createResult = mInodeTree.createPath(rpcContext, inodePath, context);
        InodeDirectory inodeDirectory = inodePath.getInode().asDirectory();
        String ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
        if (inodeDirectory.isPersisted()) {
            UfsStatus ufsStatus = context.getUfsStatus();
            // Retrieve the UFS fingerprint for this file.
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            AlluxioURI resolvedUri = resolution.getUri();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                if (ufsStatus == null) {
                    ufsFingerprint = ufs.getFingerprint(resolvedUri.toString());
                } else {
                    ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
                }
            }
        }
        mInodeTree.updateInode(rpcContext, UpdateInodeEntry.newBuilder().setId(inodeDirectory.getId()).setUfsFingerprint(ufsFingerprint).build());
        if (context.isPersisted()) {
            // The path exists in UFS, so it is no longer absent.
            mUfsAbsentPathCache.processExisting(inodePath.getUri());
        }
        Metrics.DIRECTORIES_CREATED.inc();
        return createResult;
    } catch (BlockInfoException e) {
        // happen.
        throw new RuntimeException(e);
    }
}

Example 4 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsJournalSnapshot method getSnapshot.

/**
 * Creates a snapshot of the journal.
 *
 * @param journal the journal
 * @return the journal snapshot
 */
public static UfsJournalSnapshot getSnapshot(UfsJournal journal) throws IOException {
    // Checkpoints.
    List<UfsJournalFile> checkpoints = new ArrayList<>();
    UfsStatus[] statuses = journal.getUfs().listStatus(journal.getCheckpointDir().toString());
    if (statuses != null) {
        for (UfsStatus status : statuses) {
            UfsJournalFile file = UfsJournalFile.decodeCheckpointFile(journal, status.getName());
            if (file != null) {
                checkpoints.add(file);
            }
        }
        Collections.sort(checkpoints);
    }
    List<UfsJournalFile> logs = new ArrayList<>();
    statuses = journal.getUfs().listStatus(journal.getLogDir().toString());
    if (statuses != null) {
        for (UfsStatus status : statuses) {
            UfsJournalFile file = UfsJournalFile.decodeLogFile(journal, status.getName());
            if (file != null) {
                logs.add(file);
            }
        }
        Collections.sort(logs);
    }
    List<UfsJournalFile> tmpCheckpoints = new ArrayList<>();
    statuses = journal.getUfs().listStatus(journal.getTmpDir().toString());
    if (statuses != null) {
        for (UfsStatus status : statuses) {
            tmpCheckpoints.add(UfsJournalFile.decodeTemporaryCheckpointFile(journal, status.getName()));
        }
    }
    return new UfsJournalSnapshot(checkpoints, logs, tmpCheckpoints);
}

Example 5 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsJournal method format.

/**
 * Formats the journal.
 */
public void format() throws IOException {
    URI location = getLocation();
    LOG.info("Formatting {}", location);
    if (mUfs.isDirectory(location.toString())) {
        for (UfsStatus status : mUfs.listStatus(location.toString())) {
            String childPath = URIUtils.appendPathOrDie(location, status.getName()).toString();
            if (status.isDirectory() && !mUfs.deleteDirectory(childPath, DeleteOptions.defaults().setRecursive(true)) || status.isFile() && !mUfs.deleteFile(childPath)) {
                throw new IOException(String.format("Failed to delete %s", childPath));
            }
        }
    } else if (!mUfs.mkdirs(location.toString())) {
        throw new IOException(String.format("Failed to create %s", location));
    }
    // Create a breadcrumb that indicates that the journal folder has been formatted.
    UnderFileSystemUtils.touch(mUfs, URIUtils.appendPathOrDie(location, ServerConfiguration.getString(PropertyKey.MASTER_FORMAT_FILE_PREFIX) + System.currentTimeMillis()).toString());
}