Search in sources :

Example 36 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class InodeSyncStream method loadMetadata.

/**
 * This method creates inodes containing the metadata from the UFS. The {@link UfsStatus} object
 * must be set in the {@link LoadMetadataContext} in order to successfully create the inodes.
 */
private void loadMetadata(LockedInodePath inodePath, LoadMetadataContext context) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException {
    AlluxioURI path = inodePath.getUri();
    MountTable.Resolution resolution = mMountTable.resolve(path);
    int failedSync = 0;
    try {
        if (context.getUfsStatus() == null) {
            // uri does not exist in ufs
            Inode inode = inodePath.getInode();
            if (inode.isFile()) {
                throw new IllegalArgumentException(String.format("load metadata cannot be called on a file if no ufs " + "status is present in the context. %s", inodePath.getUri()));
            }
            mInodeTree.setDirectChildrenLoaded(mRpcContext, inode.asDirectory());
            return;
        }
        if (context.getUfsStatus().isFile()) {
            loadFileMetadataInternal(mRpcContext, inodePath, resolution, context, mFsMaster);
        } else {
            loadDirectoryMetadata(mRpcContext, inodePath, context, mMountTable, mFsMaster);
            // now load all children if required
            LoadDescendantPType type = context.getOptions().getLoadDescendantType();
            if (type != LoadDescendantPType.NONE) {
                Collection<UfsStatus> children = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
                if (children == null) {
                    LOG.debug("fetching children for {} returned null", inodePath.getUri());
                    return;
                }
                for (UfsStatus childStatus : children) {
                    if (PathUtils.isTemporaryFileName(childStatus.getName())) {
                        continue;
                    }
                    AlluxioURI childURI = new AlluxioURI(PathUtils.concatPath(inodePath.getUri(), childStatus.getName()));
                    if (mInodeTree.inodePathExists(childURI) && (childStatus.isFile() || context.getOptions().getLoadDescendantType() != LoadDescendantPType.ALL)) {
                        // loading all descendants.
                        continue;
                    }
                    LoadMetadataContext loadMetadataContext = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setLoadDescendantType(LoadDescendantPType.NONE).setCommonOptions(NO_TTL_OPTION).setCreateAncestors(false)).setUfsStatus(childStatus);
                    try (LockedInodePath descendant = inodePath.lockDescendant(inodePath.getUri().joinUnsafe(childStatus.getName()), LockPattern.READ)) {
                        loadMetadata(descendant, loadMetadataContext);
                    } catch (FileNotFoundException e) {
                        LOG.debug("Failed to loadMetadata because file is not in ufs:" + " inodePath={}, options={}.", childURI, loadMetadataContext, e);
                    } catch (BlockInfoException | FileAlreadyCompletedException | FileDoesNotExistException | InvalidFileSizeException | IOException e) {
                        LOG.debug("Failed to loadMetadata because the ufs file or directory" + " is {}, options={}.", childStatus, loadMetadataContext, e);
                        failedSync++;
                    }
                }
                mInodeTree.setDirectChildrenLoaded(mRpcContext, inodePath.getInode().asDirectory());
            }
        }
    } catch (IOException | InterruptedException e) {
        LOG.debug("Failed to loadMetadata: inodePath={}, context={}.", inodePath.getUri(), context, e);
        throw new IOException(e);
    }
    if (failedSync > 0) {
        throw new IOException(String.format("Failed to load metadata of %s files or directories " + "under %s", failedSync, path));
    }
}
Also used : FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) UfsStatus(alluxio.underfs.UfsStatus) FileNotFoundException(java.io.FileNotFoundException) BlockInfoException(alluxio.exception.BlockInfoException) IOException(java.io.IOException) MountTable(alluxio.master.file.meta.MountTable) LoadDescendantPType(alluxio.grpc.LoadDescendantPType) Fingerprint(alluxio.underfs.Fingerprint) LockedInodePath(alluxio.master.file.meta.LockedInodePath) Inode(alluxio.master.file.meta.Inode) LoadMetadataContext(alluxio.master.file.contexts.LoadMetadataContext) InvalidFileSizeException(alluxio.exception.InvalidFileSizeException) FileAlreadyCompletedException(alluxio.exception.FileAlreadyCompletedException) AlluxioURI(alluxio.AlluxioURI)

Example 37 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsSyncChecker method getChildrenInUFS.

/**
 * Get the children in under storage for given alluxio path.
 *
 * @param alluxioUri alluxio path
 * @return the list of children in under storage
 * @throws InvalidPathException if aluxioUri is invalid
 * @throws IOException if a non-alluxio error occurs
 */
private UfsStatus[] getChildrenInUFS(AlluxioURI alluxioUri) throws InvalidPathException, IOException {
    MountTable.Resolution resolution = mMountTable.resolve(alluxioUri);
    AlluxioURI ufsUri = resolution.getUri();
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        AlluxioURI curUri = ufsUri;
        while (curUri != null) {
            if (mListedDirectories.containsKey(curUri.toString())) {
                List<UfsStatus> childrenList = new ArrayList<>();
                for (UfsStatus childStatus : mListedDirectories.get(curUri.toString())) {
                    String childPath = PathUtils.concatPath(curUri, childStatus.getName());
                    String prefix = PathUtils.normalizePath(ufsUri.toString(), AlluxioURI.SEPARATOR);
                    if (childPath.startsWith(prefix) && childPath.length() > prefix.length()) {
                        UfsStatus newStatus = childStatus.copy();
                        newStatus.setName(childPath.substring(prefix.length()));
                        childrenList.add(newStatus);
                    }
                }
                return trimIndirect(childrenList.toArray(new UfsStatus[childrenList.size()]));
            }
            curUri = curUri.getParent();
        }
        UfsStatus[] children = ufs.listStatus(ufsUri.toString(), ListOptions.defaults().setRecursive(true));
        // Assumption: multiple mounted UFSs cannot have the same ufsUri
        if (children == null) {
            return EMPTY_CHILDREN;
        }
        mListedDirectories.put(ufsUri.toString(), children);
        return trimIndirect(children);
    }
}
Also used : UfsStatus(alluxio.underfs.UfsStatus) ArrayList(java.util.ArrayList) MountTable(alluxio.master.file.meta.MountTable) UnderFileSystem(alluxio.underfs.UnderFileSystem) AlluxioURI(alluxio.AlluxioURI)

Example 38 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsSyncChecker method checkDirectory.

/**
 * Check if immediate children of directory are in sync with UFS.
 *
 * @param inode read-locked directory to check
 * @param alluxioUri path of directory to to check
 */
@SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
public void checkDirectory(InodeDirectory inode, AlluxioURI alluxioUri) throws FileDoesNotExistException, InvalidPathException, IOException {
    Preconditions.checkArgument(inode.isPersisted());
    UfsStatus[] ufsChildren = getChildrenInUFS(alluxioUri);
    // Filter out temporary files
    ufsChildren = Arrays.stream(ufsChildren).filter(ufsStatus -> !PathUtils.isTemporaryFileName(ufsStatus.getName())).toArray(UfsStatus[]::new);
    Arrays.sort(ufsChildren, Comparator.comparing(UfsStatus::getName));
    Inode[] alluxioChildren = Iterables.toArray(mInodeStore.getChildren(inode), Inode.class);
    Arrays.sort(alluxioChildren);
    int ufsPos = 0;
    for (Inode alluxioInode : alluxioChildren) {
        if (ufsPos >= ufsChildren.length) {
            break;
        }
        String ufsName = ufsChildren[ufsPos].getName();
        if (ufsName.endsWith(AlluxioURI.SEPARATOR)) {
            ufsName = ufsName.substring(0, ufsName.length() - 1);
        }
        if (ufsName.equals(alluxioInode.getName())) {
            ufsPos++;
        }
    }
    if (ufsPos == ufsChildren.length) {
        // Directory is in sync
        mSyncedDirectories.put(alluxioUri, inode);
    } else {
        // Invalidate ancestor directories if not a mount point
        AlluxioURI currentPath = alluxioUri;
        while (currentPath.getParent() != null && !mMountTable.isMountPoint(currentPath) && mSyncedDirectories.containsKey(currentPath.getParent())) {
            mSyncedDirectories.remove(currentPath.getParent());
            currentPath = currentPath.getParent();
        }
        LOG.debug("Ufs file {} does not match any file in Alluxio", ufsChildren[ufsPos]);
    }
}
Also used : Inode(alluxio.master.file.meta.Inode) UfsStatus(alluxio.underfs.UfsStatus) AlluxioURI(alluxio.AlluxioURI) SuppressFBWarnings(alluxio.annotation.SuppressFBWarnings)

Example 39 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsDirectoryValidationTask method validateImpl.

@Override
public ValidationTaskResult validateImpl(Map<String, String> optionsMap) {
    StringBuilder msg = new StringBuilder();
    StringBuilder advice = new StringBuilder();
    try {
        UnderFileSystem ufs = UnderFileSystem.Factory.create(mPath, mConf);
        UfsStatus[] listStatus = ufs.listStatus(mPath);
        if (listStatus == null) {
            msg.append(String.format("Unable to list under file system path %s. ", mPath));
            advice.append(String.format("Please check if path %s denotes a directory. ", mPath));
            return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
        }
        msg.append(String.format("Successfully listed path %s. ", mPath));
        return new ValidationTaskResult(ValidationUtils.State.OK, getName(), msg.toString(), advice.toString());
    } catch (Exception e) {
        msg.append(String.format("Unable to access under file system path %s: %s. ", mPath, e.getMessage()));
        msg.append(ValidationUtils.getErrorInfo(e));
        advice.append(String.format("Please verify your path %s is correct.%n", mPath));
        return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
    }
}
Also used : UfsStatus(alluxio.underfs.UfsStatus) UnderFileSystem(alluxio.underfs.UnderFileSystem)

Example 40 with UfsStatus

use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.

the class UfsSuperUserValidationTask method validateImpl.

@Override
public ValidationTaskResult validateImpl(Map<String, String> optionsMap) {
    StringBuilder msg = new StringBuilder();
    StringBuilder advice = new StringBuilder();
    if (!ValidationUtils.isHdfsScheme(mPath)) {
        // only support check on HDFS for now
        msg.append(String.format("Under file system is not HDFS. Skip validation. "));
        return new ValidationTaskResult(ValidationUtils.State.SKIPPED, getName(), msg.toString(), advice.toString());
    }
    UfsStatus status;
    UnderFileSystem ufs;
    try {
        ufs = UnderFileSystem.Factory.create(mPath, mConf);
        status = ufs.getStatus(mPath);
        if (status == null) {
            msg.append(String.format("Unable to get status for under file system path %s. ", mPath));
            advice.append(String.format("Please check your path %s. ", mPath));
            return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
        }
        if (Strings.isNullOrEmpty(status.getOwner()) && Strings.isNullOrEmpty(status.getGroup())) {
            msg.append(String.format("Cannot determine owner of under file system path %s. ", mPath));
            advice.append(String.format("Please check your path %s. ", mPath));
            return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
        }
    } catch (Exception e) {
        msg.append(String.format("Unable to access under file system path %s: %s.", mPath, e.getMessage()));
        msg.append(ValidationUtils.getErrorInfo(e));
        return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
    }
    try {
        ufs.setOwner(mPath, status.getOwner(), status.getGroup());
        msg.append(String.format("User has superuser privilege to path %s.%n", mPath));
        return new ValidationTaskResult(ValidationUtils.State.OK, getName(), msg.toString(), advice.toString());
    } catch (IOException e) {
        msg.append(String.format("Unable to set owner of under file system path %s: %s. ", mPath, e.getMessage()));
        advice.append("Please check if Alluxio is super user on the file system. ");
        return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
    }
}
Also used : UfsStatus(alluxio.underfs.UfsStatus) IOException(java.io.IOException) UnderFileSystem(alluxio.underfs.UnderFileSystem) IOException(java.io.IOException)

Aggregations

UfsStatus (alluxio.underfs.UfsStatus)40 UnderFileSystem (alluxio.underfs.UnderFileSystem)12 IOException (java.io.IOException)12 Test (org.junit.Test)11 BaseIntegrationTest (alluxio.testutils.BaseIntegrationTest)10 Path (org.apache.hadoop.fs.Path)9 AlluxioURI (alluxio.AlluxioURI)8 Inode (alluxio.master.file.meta.Inode)7 MountTable (alluxio.master.file.meta.MountTable)7 UfsFileStatus (alluxio.underfs.UfsFileStatus)6 UfsDirectoryStatus (alluxio.underfs.UfsDirectoryStatus)5 ArrayList (java.util.ArrayList)5 FileStatus (org.apache.hadoop.fs.FileStatus)5 BlockInfoException (alluxio.exception.BlockInfoException)4 FileDoesNotExistException (alluxio.exception.FileDoesNotExistException)4 FileNotFoundException (java.io.FileNotFoundException)4 Fingerprint (alluxio.underfs.Fingerprint)3 URI (java.net.URI)3 InvalidPathException (alluxio.exception.InvalidPathException)2 LoadMetadataContext (alluxio.master.file.contexts.LoadMetadataContext)2