Examples with UfsStatus alluxio.underfs.UfsStatus used on opensource projects

Example 6 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class InodeSyncStream method syncExistingInodeMetadata.

/**
 * Sync inode metadata with the UFS state.
 *
 * This method expects the {@code inodePath} to already exist in the inode tree.
 */
private void syncExistingInodeMetadata(LockedInodePath inodePath, boolean skipLoad) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException, InterruptedException {
    if (inodePath.getLockPattern() != LockPattern.WRITE_EDGE && !mLoadOnly) {
        throw new RuntimeException(String.format("syncExistingInodeMetadata was called on d%s when only locked with %s. Load metadata" + " only was not specified.", inodePath.getUri(), inodePath.getLockPattern()));
    }
    // Set to true if the given inode was deleted.
    boolean deletedInode = false;
    // whether we need to load metadata for the current path
    boolean loadMetadata = mLoadOnly;
    LOG.trace("Syncing inode metadata {}", inodePath.getUri());
    // The requested path already exists in Alluxio.
    Inode inode = inodePath.getInode();
    // initialize sync children to true if it is a listStatus call on a directory
    boolean syncChildren = inode.isDirectory() && !mIsGetFileInfo && !mDescendantType.equals(DescendantType.NONE);
    if (inodePath.getLockPattern() == LockPattern.WRITE_EDGE && !mLoadOnly) {
        if (inode instanceof InodeFile && !inode.asFile().isCompleted()) {
            // Do not sync an incomplete file, since the UFS file is expected to not exist.
            return;
        }
        Optional<Scoped> persistingLock = mInodeLockManager.tryAcquirePersistingLock(inode.getId());
        if (!persistingLock.isPresent()) {
            // written.
            return;
        }
        persistingLock.get().close();
        UfsStatus cachedStatus = null;
        boolean fileNotFound = false;
        try {
            cachedStatus = mStatusCache.getStatus(inodePath.getUri());
        } catch (FileNotFoundException e) {
            fileNotFound = true;
        }
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        AlluxioURI ufsUri = resolution.getUri();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            String ufsFingerprint;
            Fingerprint ufsFpParsed;
            // When the status is not cached and it was not due to file not found, we retry
            if (fileNotFound) {
                ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
                ufsFpParsed = Fingerprint.parse(ufsFingerprint);
            } else if (cachedStatus == null) {
                // TODO(david): change the interface so that getFingerprint returns a parsed fingerprint
                ufsFingerprint = (String) getFromUfs(() -> ufs.getFingerprint(ufsUri.toString()));
                ufsFpParsed = Fingerprint.parse(ufsFingerprint);
            } else {
                // When the status is cached
                Pair<AccessControlList, DefaultAccessControlList> aclPair = (Pair<AccessControlList, DefaultAccessControlList>) getFromUfs(() -> ufs.getAclPair(ufsUri.toString()));
                if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
                    ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus);
                } else {
                    ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus, aclPair.getFirst());
                }
                ufsFingerprint = ufsFpParsed.serialize();
            }
            boolean containsMountPoint = mMountTable.containsMountPoint(inodePath.getUri(), true);
            UfsSyncUtils.SyncPlan syncPlan = UfsSyncUtils.computeSyncPlan(inode, ufsFpParsed, containsMountPoint);
            if (syncPlan.toUpdateMetaData()) {
                // It works by calling SetAttributeInternal on the inodePath.
                if (ufsFpParsed != null && ufsFpParsed.isValid()) {
                    short mode = Short.parseShort(ufsFpParsed.getTag(Fingerprint.Tag.MODE));
                    long opTimeMs = System.currentTimeMillis();
                    SetAttributePOptions.Builder builder = SetAttributePOptions.newBuilder().setMode(new Mode(mode).toProto());
                    String owner = ufsFpParsed.getTag(Fingerprint.Tag.OWNER);
                    if (!owner.equals(Fingerprint.UNDERSCORE)) {
                        // Only set owner if not empty
                        builder.setOwner(owner);
                    }
                    String group = ufsFpParsed.getTag(Fingerprint.Tag.GROUP);
                    if (!group.equals(Fingerprint.UNDERSCORE)) {
                        // Only set group if not empty
                        builder.setGroup(group);
                    }
                    SetAttributeContext ctx = SetAttributeContext.mergeFrom(builder).setUfsFingerprint(ufsFingerprint);
                    mFsMaster.setAttributeSingleFile(mRpcContext, inodePath, false, opTimeMs, ctx);
                }
            }
            if (syncPlan.toDelete()) {
                deletedInode = true;
                try {
                    // The options for deleting.
                    DeleteContext syncDeleteContext = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(true).setUnchecked(true));
                    mFsMaster.deleteInternal(mRpcContext, inodePath, syncDeleteContext);
                } catch (DirectoryNotEmptyException | IOException e) {
                    // Should not happen, since it is an unchecked delete.
                    LOG.error("Unexpected error for unchecked delete.", e);
                }
            }
            if (syncPlan.toLoadMetadata()) {
                loadMetadata = true;
            }
            syncChildren = syncPlan.toSyncChildren();
        }
    }
    // if DescendantType.ONE we only sync children if we are syncing root of this stream
    if (mDescendantType == DescendantType.ONE) {
        syncChildren = syncChildren && mRootScheme.getPath().equals(inodePath.getUri());
    }
    Map<String, Inode> inodeChildren = new HashMap<>();
    if (syncChildren) {
        // maps children name to inode
        mInodeStore.getChildren(inode.asDirectory()).forEach(child -> inodeChildren.put(child.getName(), child));
        // Fetch and populate children into the cache
        mStatusCache.prefetchChildren(inodePath.getUri(), mMountTable);
        Collection<UfsStatus> listStatus = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
        // Iterate over UFS listings and process UFS children.
        if (listStatus != null) {
            for (UfsStatus ufsChildStatus : listStatus) {
                if (!inodeChildren.containsKey(ufsChildStatus.getName()) && !PathUtils.isTemporaryFileName(ufsChildStatus.getName())) {
                    // Ufs child exists, but Alluxio child does not. Must load metadata.
                    loadMetadata = true;
                    break;
                }
            }
        }
    }
    // locked path
    if (deletedInode) {
        inodePath.removeLastInode();
    }
    // load metadata if necessary.
    if (loadMetadata && !skipLoad) {
        loadMetadataForPath(inodePath);
    }
    if (syncChildren) {
        // Iterate over Alluxio children and process persisted children.
        mInodeStore.getChildren(inode.asDirectory()).forEach(childInode -> {
            // its children.
            if (mLoadOnly && inodeChildren.containsKey(childInode.getName()) && childInode.isFile()) {
                return;
            }
            // If we're performing a recursive sync, add each child of our current Inode to the queue
            AlluxioURI child = inodePath.getUri().joinUnsafe(childInode.getName());
            mPendingPaths.add(child);
            // This asynchronously schedules a job to pre-fetch the statuses into the cache.
            if (childInode.isDirectory() && mDescendantType == DescendantType.ALL) {
                mStatusCache.prefetchChildren(child, mMountTable);
            }
        });
    }
}

Example 7 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class DailyMetadataBackup method deleteStaleBackups.

/**
 * Deletes stale backup files to avoid consuming too many spaces.
 */
private void deleteStaleBackups() throws Exception {
    try (CloseableResource<UnderFileSystem> ufsResource = mUfsManager.getRoot().acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        UfsStatus[] statuses = ufs.listStatus(mBackupDir);
        if (statuses.length <= mRetainedFiles) {
            return;
        }
        // Sort the backup files according to create time from oldest to newest
        TreeMap<Instant, String> timeToFile = new TreeMap<>((a, b) -> (a.isBefore(b) ? -1 : a.isAfter(b) ? 1 : 0));
        for (UfsStatus status : statuses) {
            if (status.isFile()) {
                Matcher matcher = BackupManager.BACKUP_FILE_PATTERN.matcher(status.getName());
                if (matcher.matches()) {
                    timeToFile.put(Instant.ofEpochMilli(Long.parseLong(matcher.group(1))), status.getName());
                }
            }
        }
        int toDeleteFileNum = timeToFile.size() - mRetainedFiles;
        if (toDeleteFileNum <= 0) {
            return;
        }
        for (int i = 0; i < toDeleteFileNum; i++) {
            String toDeleteFile = PathUtils.concatPath(mBackupDir, timeToFile.pollFirstEntry().getValue());
            ufs.deleteExistingFile(toDeleteFile);
        }
        LOG.info("Deleted {} stale metadata backup files at {}", toDeleteFileNum, mBackupDir);
    }
}

Example 8 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UnderFileSystemCommonOperations method getFileStatusTest.

/**
 * Test for getting file status.
 */
@RelatedS3Operations(operations = { "upload", "getObjectMetadata" })
public void getFileStatusTest() throws IOException {
    String testFile = PathUtils.concatPath(mTopLevelTestDirectory, "testFile");
    createEmptyFile(testFile);
    UfsStatus status = mUfs.getStatus(testFile);
    if (!(status instanceof UfsFileStatus)) {
        throw new IOException("Failed to get ufs file status");
    }
}

Example 9 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class S3ASpecificOperations method createMultipartFileTest.

/**
 * Test for creating a multipart file.
 */
@RelatedS3Operations(operations = { "initiateMultipartUpload", "uploadPart", "completeMultipartUpload", "listObjects" })
public void createMultipartFileTest() throws IOException {
    String testParent = PathUtils.concatPath(mTestDirectory, "createParent");
    String testFile = PathUtils.concatPath(testParent, "createMultipart");
    int numCopies;
    try (OutputStream outputStream = mUfs.create(testFile, CreateOptions.defaults(mConfiguration).setCreateParent(true))) {
        numCopies = 6 * Constants.MB / TEST_BYTES.length;
        for (int i = 0; i < numCopies; ++i) {
            outputStream.write(TEST_BYTES);
        }
    }
    // Validate data written successfully and content is correct
    try (InputStream inputStream = mUfs.openExistingFile(testFile)) {
        byte[] buf = new byte[numCopies * TEST_BYTES.length];
        int offset = 0;
        while (offset < buf.length) {
            int bytesRead = inputStream.read(buf, offset, buf.length - offset);
            for (int i = 0; i < bytesRead; ++i) {
                if (TEST_BYTES[(offset + i) % TEST_BYTES.length] != buf[offset + i]) {
                    throw new IOException("Content of the written file is incorrect");
                }
            }
            offset += bytesRead;
        }
    }
    // These two calls will trigger S3A list objects v1
    if (!mUfs.isDirectory(testParent)) {
        throw new IOException("The written directory does not exist or is file");
    }
    UfsStatus[] statuses = mUfs.listStatus(PathUtils.concatPath(mTestDirectory, "createParent"));
    if (statuses.length != 1) {
        throw new IOException("List status result is incorrect");
    }
}

Example 10 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class HdfsUnderFileSystem method listStatus.

@Override
@Nullable
public UfsStatus[] listStatus(String path) throws IOException {
    FileStatus[] files = listStatusInternal(path);
    if (files == null) {
        return null;
    }
    UfsStatus[] rtn = new UfsStatus[files.length];
    int i = 0;
    for (FileStatus status : files) {
        // only return the relative path, to keep consistent with java.io.File.list()
        UfsStatus retStatus;
        if (!status.isDir()) {
            String contentHash = UnderFileSystemUtils.approximateContentHash(status.getLen(), status.getModificationTime());
            retStatus = new UfsFileStatus(status.getPath().getName(), contentHash, status.getLen(), status.getModificationTime(), status.getOwner(), status.getGroup(), status.getPermission().toShort(), status.getBlockSize());
        } else {
            retStatus = new UfsDirectoryStatus(status.getPath().getName(), status.getOwner(), status.getGroup(), status.getPermission().toShort(), status.getModificationTime());
        }
        rtn[i++] = retStatus;
    }
    return rtn;
}