Examples with UfsStatus alluxio.underfs.UfsStatus used on opensource projects

Example 36 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class InodeSyncStream method loadMetadata.

/**
 * This method creates inodes containing the metadata from the UFS. The {@link UfsStatus} object
 * must be set in the {@link LoadMetadataContext} in order to successfully create the inodes.
 */
private void loadMetadata(LockedInodePath inodePath, LoadMetadataContext context) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException {
    AlluxioURI path = inodePath.getUri();
    MountTable.Resolution resolution = mMountTable.resolve(path);
    int failedSync = 0;
    try {
        if (context.getUfsStatus() == null) {
            // uri does not exist in ufs
            Inode inode = inodePath.getInode();
            if (inode.isFile()) {
                throw new IllegalArgumentException(String.format("load metadata cannot be called on a file if no ufs " + "status is present in the context. %s", inodePath.getUri()));
            }
            mInodeTree.setDirectChildrenLoaded(mRpcContext, inode.asDirectory());
            return;
        }
        if (context.getUfsStatus().isFile()) {
            loadFileMetadataInternal(mRpcContext, inodePath, resolution, context, mFsMaster);
        } else {
            loadDirectoryMetadata(mRpcContext, inodePath, context, mMountTable, mFsMaster);
            // now load all children if required
            LoadDescendantPType type = context.getOptions().getLoadDescendantType();
            if (type != LoadDescendantPType.NONE) {
                Collection<UfsStatus> children = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
                if (children == null) {
                    LOG.debug("fetching children for {} returned null", inodePath.getUri());
                    return;
                }
                for (UfsStatus childStatus : children) {
                    if (PathUtils.isTemporaryFileName(childStatus.getName())) {
                        continue;
                    }
                    AlluxioURI childURI = new AlluxioURI(PathUtils.concatPath(inodePath.getUri(), childStatus.getName()));
                    if (mInodeTree.inodePathExists(childURI) && (childStatus.isFile() || context.getOptions().getLoadDescendantType() != LoadDescendantPType.ALL)) {
                        // loading all descendants.
                        continue;
                    }
                    LoadMetadataContext loadMetadataContext = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setLoadDescendantType(LoadDescendantPType.NONE).setCommonOptions(NO_TTL_OPTION).setCreateAncestors(false)).setUfsStatus(childStatus);
                    try (LockedInodePath descendant = inodePath.lockDescendant(inodePath.getUri().joinUnsafe(childStatus.getName()), LockPattern.READ)) {
                        loadMetadata(descendant, loadMetadataContext);
                    } catch (FileNotFoundException e) {
                        LOG.debug("Failed to loadMetadata because file is not in ufs:" + " inodePath={}, options={}.", childURI, loadMetadataContext, e);
                    } catch (BlockInfoException | FileAlreadyCompletedException | FileDoesNotExistException | InvalidFileSizeException | IOException e) {
                        LOG.debug("Failed to loadMetadata because the ufs file or directory" + " is {}, options={}.", childStatus, loadMetadataContext, e);
                        failedSync++;
                    }
                }
                mInodeTree.setDirectChildrenLoaded(mRpcContext, inodePath.getInode().asDirectory());
            }
        }
    } catch (IOException | InterruptedException e) {
        LOG.debug("Failed to loadMetadata: inodePath={}, context={}.", inodePath.getUri(), context, e);
        throw new IOException(e);
    }
    if (failedSync > 0) {
        throw new IOException(String.format("Failed to load metadata of %s files or directories " + "under %s", failedSync, path));
    }
}

Example 37 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsSyncChecker method getChildrenInUFS.

/**
 * Get the children in under storage for given alluxio path.
 *
 * @param alluxioUri alluxio path
 * @return the list of children in under storage
 * @throws InvalidPathException if aluxioUri is invalid
 * @throws IOException if a non-alluxio error occurs
 */
private UfsStatus[] getChildrenInUFS(AlluxioURI alluxioUri) throws InvalidPathException, IOException {
    MountTable.Resolution resolution = mMountTable.resolve(alluxioUri);
    AlluxioURI ufsUri = resolution.getUri();
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        AlluxioURI curUri = ufsUri;
        while (curUri != null) {
            if (mListedDirectories.containsKey(curUri.toString())) {
                List<UfsStatus> childrenList = new ArrayList<>();
                for (UfsStatus childStatus : mListedDirectories.get(curUri.toString())) {
                    String childPath = PathUtils.concatPath(curUri, childStatus.getName());
                    String prefix = PathUtils.normalizePath(ufsUri.toString(), AlluxioURI.SEPARATOR);
                    if (childPath.startsWith(prefix) && childPath.length() > prefix.length()) {
                        UfsStatus newStatus = childStatus.copy();
                        newStatus.setName(childPath.substring(prefix.length()));
                        childrenList.add(newStatus);
                    }
                }
                return trimIndirect(childrenList.toArray(new UfsStatus[childrenList.size()]));
            }
            curUri = curUri.getParent();
        }
        UfsStatus[] children = ufs.listStatus(ufsUri.toString(), ListOptions.defaults().setRecursive(true));
        // Assumption: multiple mounted UFSs cannot have the same ufsUri
        if (children == null) {
            return EMPTY_CHILDREN;
        }
        mListedDirectories.put(ufsUri.toString(), children);
        return trimIndirect(children);
    }
}

Example 38 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsSyncChecker method checkDirectory.

/**
 * Check if immediate children of directory are in sync with UFS.
 *
 * @param inode read-locked directory to check
 * @param alluxioUri path of directory to to check
 */
@SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
public void checkDirectory(InodeDirectory inode, AlluxioURI alluxioUri) throws FileDoesNotExistException, InvalidPathException, IOException {
    Preconditions.checkArgument(inode.isPersisted());
    UfsStatus[] ufsChildren = getChildrenInUFS(alluxioUri);
    // Filter out temporary files
    ufsChildren = Arrays.stream(ufsChildren).filter(ufsStatus -> !PathUtils.isTemporaryFileName(ufsStatus.getName())).toArray(UfsStatus[]::new);
    Arrays.sort(ufsChildren, Comparator.comparing(UfsStatus::getName));
    Inode[] alluxioChildren = Iterables.toArray(mInodeStore.getChildren(inode), Inode.class);
    Arrays.sort(alluxioChildren);
    int ufsPos = 0;
    for (Inode alluxioInode : alluxioChildren) {
        if (ufsPos >= ufsChildren.length) {
            break;
        }
        String ufsName = ufsChildren[ufsPos].getName();
        if (ufsName.endsWith(AlluxioURI.SEPARATOR)) {
            ufsName = ufsName.substring(0, ufsName.length() - 1);
        }
        if (ufsName.equals(alluxioInode.getName())) {
            ufsPos++;
        }
    }
    if (ufsPos == ufsChildren.length) {
        // Directory is in sync
        mSyncedDirectories.put(alluxioUri, inode);
    } else {
        // Invalidate ancestor directories if not a mount point
        AlluxioURI currentPath = alluxioUri;
        while (currentPath.getParent() != null && !mMountTable.isMountPoint(currentPath) && mSyncedDirectories.containsKey(currentPath.getParent())) {
            mSyncedDirectories.remove(currentPath.getParent());
            currentPath = currentPath.getParent();
        }
        LOG.debug("Ufs file {} does not match any file in Alluxio", ufsChildren[ufsPos]);
    }
}

Example 39 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsDirectoryValidationTask method validateImpl.

@Override
public ValidationTaskResult validateImpl(Map<String, String> optionsMap) {
    StringBuilder msg = new StringBuilder();
    StringBuilder advice = new StringBuilder();
    try {
        UnderFileSystem ufs = UnderFileSystem.Factory.create(mPath, mConf);
        UfsStatus[] listStatus = ufs.listStatus(mPath);
        if (listStatus == null) {
            msg.append(String.format("Unable to list under file system path %s. ", mPath));
            advice.append(String.format("Please check if path %s denotes a directory. ", mPath));
            return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
        }
        msg.append(String.format("Successfully listed path %s. ", mPath));
        return new ValidationTaskResult(ValidationUtils.State.OK, getName(), msg.toString(), advice.toString());
    } catch (Exception e) {
        msg.append(String.format("Unable to access under file system path %s: %s. ", mPath, e.getMessage()));
        msg.append(ValidationUtils.getErrorInfo(e));
        advice.append(String.format("Please verify your path %s is correct.%n", mPath));
        return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
    }
}

Example 40 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsSuperUserValidationTask method validateImpl.

@Override
public ValidationTaskResult validateImpl(Map<String, String> optionsMap) {
    StringBuilder msg = new StringBuilder();
    StringBuilder advice = new StringBuilder();
    if (!ValidationUtils.isHdfsScheme(mPath)) {
        // only support check on HDFS for now
        msg.append(String.format("Under file system is not HDFS. Skip validation. "));
        return new ValidationTaskResult(ValidationUtils.State.SKIPPED, getName(), msg.toString(), advice.toString());
    }
    UfsStatus status;
    UnderFileSystem ufs;
    try {
        ufs = UnderFileSystem.Factory.create(mPath, mConf);
        status = ufs.getStatus(mPath);
        if (status == null) {
            msg.append(String.format("Unable to get status for under file system path %s. ", mPath));
            advice.append(String.format("Please check your path %s. ", mPath));
            return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
        }
        if (Strings.isNullOrEmpty(status.getOwner()) && Strings.isNullOrEmpty(status.getGroup())) {
            msg.append(String.format("Cannot determine owner of under file system path %s. ", mPath));
            advice.append(String.format("Please check your path %s. ", mPath));
            return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
        }
    } catch (Exception e) {
        msg.append(String.format("Unable to access under file system path %s: %s.", mPath, e.getMessage()));
        msg.append(ValidationUtils.getErrorInfo(e));
        return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
    }
    try {
        ufs.setOwner(mPath, status.getOwner(), status.getGroup());
        msg.append(String.format("User has superuser privilege to path %s.%n", mPath));
        return new ValidationTaskResult(ValidationUtils.State.OK, getName(), msg.toString(), advice.toString());
    } catch (IOException e) {
        msg.append(String.format("Unable to set owner of under file system path %s: %s. ", mPath, e.getMessage()));
        advice.append("Please check if Alluxio is super user on the file system. ");
        return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
    }
}