use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.
the class InodeSyncStream method loadMetadata.
/**
* This method creates inodes containing the metadata from the UFS. The {@link UfsStatus} object
* must be set in the {@link LoadMetadataContext} in order to successfully create the inodes.
*/
private void loadMetadata(LockedInodePath inodePath, LoadMetadataContext context) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException {
AlluxioURI path = inodePath.getUri();
MountTable.Resolution resolution = mMountTable.resolve(path);
int failedSync = 0;
try {
if (context.getUfsStatus() == null) {
// uri does not exist in ufs
Inode inode = inodePath.getInode();
if (inode.isFile()) {
throw new IllegalArgumentException(String.format("load metadata cannot be called on a file if no ufs " + "status is present in the context. %s", inodePath.getUri()));
}
mInodeTree.setDirectChildrenLoaded(mRpcContext, inode.asDirectory());
return;
}
if (context.getUfsStatus().isFile()) {
loadFileMetadataInternal(mRpcContext, inodePath, resolution, context, mFsMaster);
} else {
loadDirectoryMetadata(mRpcContext, inodePath, context, mMountTable, mFsMaster);
// now load all children if required
LoadDescendantPType type = context.getOptions().getLoadDescendantType();
if (type != LoadDescendantPType.NONE) {
Collection<UfsStatus> children = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
if (children == null) {
LOG.debug("fetching children for {} returned null", inodePath.getUri());
return;
}
for (UfsStatus childStatus : children) {
if (PathUtils.isTemporaryFileName(childStatus.getName())) {
continue;
}
AlluxioURI childURI = new AlluxioURI(PathUtils.concatPath(inodePath.getUri(), childStatus.getName()));
if (mInodeTree.inodePathExists(childURI) && (childStatus.isFile() || context.getOptions().getLoadDescendantType() != LoadDescendantPType.ALL)) {
// loading all descendants.
continue;
}
LoadMetadataContext loadMetadataContext = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setLoadDescendantType(LoadDescendantPType.NONE).setCommonOptions(NO_TTL_OPTION).setCreateAncestors(false)).setUfsStatus(childStatus);
try (LockedInodePath descendant = inodePath.lockDescendant(inodePath.getUri().joinUnsafe(childStatus.getName()), LockPattern.READ)) {
loadMetadata(descendant, loadMetadataContext);
} catch (FileNotFoundException e) {
LOG.debug("Failed to loadMetadata because file is not in ufs:" + " inodePath={}, options={}.", childURI, loadMetadataContext, e);
} catch (BlockInfoException | FileAlreadyCompletedException | FileDoesNotExistException | InvalidFileSizeException | IOException e) {
LOG.debug("Failed to loadMetadata because the ufs file or directory" + " is {}, options={}.", childStatus, loadMetadataContext, e);
failedSync++;
}
}
mInodeTree.setDirectChildrenLoaded(mRpcContext, inodePath.getInode().asDirectory());
}
}
} catch (IOException | InterruptedException e) {
LOG.debug("Failed to loadMetadata: inodePath={}, context={}.", inodePath.getUri(), context, e);
throw new IOException(e);
}
if (failedSync > 0) {
throw new IOException(String.format("Failed to load metadata of %s files or directories " + "under %s", failedSync, path));
}
}
use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.
the class UfsSyncChecker method getChildrenInUFS.
/**
* Get the children in under storage for given alluxio path.
*
* @param alluxioUri alluxio path
* @return the list of children in under storage
* @throws InvalidPathException if aluxioUri is invalid
* @throws IOException if a non-alluxio error occurs
*/
private UfsStatus[] getChildrenInUFS(AlluxioURI alluxioUri) throws InvalidPathException, IOException {
MountTable.Resolution resolution = mMountTable.resolve(alluxioUri);
AlluxioURI ufsUri = resolution.getUri();
try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
UnderFileSystem ufs = ufsResource.get();
AlluxioURI curUri = ufsUri;
while (curUri != null) {
if (mListedDirectories.containsKey(curUri.toString())) {
List<UfsStatus> childrenList = new ArrayList<>();
for (UfsStatus childStatus : mListedDirectories.get(curUri.toString())) {
String childPath = PathUtils.concatPath(curUri, childStatus.getName());
String prefix = PathUtils.normalizePath(ufsUri.toString(), AlluxioURI.SEPARATOR);
if (childPath.startsWith(prefix) && childPath.length() > prefix.length()) {
UfsStatus newStatus = childStatus.copy();
newStatus.setName(childPath.substring(prefix.length()));
childrenList.add(newStatus);
}
}
return trimIndirect(childrenList.toArray(new UfsStatus[childrenList.size()]));
}
curUri = curUri.getParent();
}
UfsStatus[] children = ufs.listStatus(ufsUri.toString(), ListOptions.defaults().setRecursive(true));
// Assumption: multiple mounted UFSs cannot have the same ufsUri
if (children == null) {
return EMPTY_CHILDREN;
}
mListedDirectories.put(ufsUri.toString(), children);
return trimIndirect(children);
}
}
use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.
the class UfsSyncChecker method checkDirectory.
/**
* Check if immediate children of directory are in sync with UFS.
*
* @param inode read-locked directory to check
* @param alluxioUri path of directory to to check
*/
@SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE")
public void checkDirectory(InodeDirectory inode, AlluxioURI alluxioUri) throws FileDoesNotExistException, InvalidPathException, IOException {
Preconditions.checkArgument(inode.isPersisted());
UfsStatus[] ufsChildren = getChildrenInUFS(alluxioUri);
// Filter out temporary files
ufsChildren = Arrays.stream(ufsChildren).filter(ufsStatus -> !PathUtils.isTemporaryFileName(ufsStatus.getName())).toArray(UfsStatus[]::new);
Arrays.sort(ufsChildren, Comparator.comparing(UfsStatus::getName));
Inode[] alluxioChildren = Iterables.toArray(mInodeStore.getChildren(inode), Inode.class);
Arrays.sort(alluxioChildren);
int ufsPos = 0;
for (Inode alluxioInode : alluxioChildren) {
if (ufsPos >= ufsChildren.length) {
break;
}
String ufsName = ufsChildren[ufsPos].getName();
if (ufsName.endsWith(AlluxioURI.SEPARATOR)) {
ufsName = ufsName.substring(0, ufsName.length() - 1);
}
if (ufsName.equals(alluxioInode.getName())) {
ufsPos++;
}
}
if (ufsPos == ufsChildren.length) {
// Directory is in sync
mSyncedDirectories.put(alluxioUri, inode);
} else {
// Invalidate ancestor directories if not a mount point
AlluxioURI currentPath = alluxioUri;
while (currentPath.getParent() != null && !mMountTable.isMountPoint(currentPath) && mSyncedDirectories.containsKey(currentPath.getParent())) {
mSyncedDirectories.remove(currentPath.getParent());
currentPath = currentPath.getParent();
}
LOG.debug("Ufs file {} does not match any file in Alluxio", ufsChildren[ufsPos]);
}
}
use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.
the class UfsDirectoryValidationTask method validateImpl.
@Override
public ValidationTaskResult validateImpl(Map<String, String> optionsMap) {
StringBuilder msg = new StringBuilder();
StringBuilder advice = new StringBuilder();
try {
UnderFileSystem ufs = UnderFileSystem.Factory.create(mPath, mConf);
UfsStatus[] listStatus = ufs.listStatus(mPath);
if (listStatus == null) {
msg.append(String.format("Unable to list under file system path %s. ", mPath));
advice.append(String.format("Please check if path %s denotes a directory. ", mPath));
return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
}
msg.append(String.format("Successfully listed path %s. ", mPath));
return new ValidationTaskResult(ValidationUtils.State.OK, getName(), msg.toString(), advice.toString());
} catch (Exception e) {
msg.append(String.format("Unable to access under file system path %s: %s. ", mPath, e.getMessage()));
msg.append(ValidationUtils.getErrorInfo(e));
advice.append(String.format("Please verify your path %s is correct.%n", mPath));
return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
}
}
use of alluxio.underfs.UfsStatus in project alluxio by Alluxio.
the class UfsSuperUserValidationTask method validateImpl.
@Override
public ValidationTaskResult validateImpl(Map<String, String> optionsMap) {
StringBuilder msg = new StringBuilder();
StringBuilder advice = new StringBuilder();
if (!ValidationUtils.isHdfsScheme(mPath)) {
// only support check on HDFS for now
msg.append(String.format("Under file system is not HDFS. Skip validation. "));
return new ValidationTaskResult(ValidationUtils.State.SKIPPED, getName(), msg.toString(), advice.toString());
}
UfsStatus status;
UnderFileSystem ufs;
try {
ufs = UnderFileSystem.Factory.create(mPath, mConf);
status = ufs.getStatus(mPath);
if (status == null) {
msg.append(String.format("Unable to get status for under file system path %s. ", mPath));
advice.append(String.format("Please check your path %s. ", mPath));
return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
}
if (Strings.isNullOrEmpty(status.getOwner()) && Strings.isNullOrEmpty(status.getGroup())) {
msg.append(String.format("Cannot determine owner of under file system path %s. ", mPath));
advice.append(String.format("Please check your path %s. ", mPath));
return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
}
} catch (Exception e) {
msg.append(String.format("Unable to access under file system path %s: %s.", mPath, e.getMessage()));
msg.append(ValidationUtils.getErrorInfo(e));
return new ValidationTaskResult(ValidationUtils.State.FAILED, getName(), msg.toString(), advice.toString());
}
try {
ufs.setOwner(mPath, status.getOwner(), status.getGroup());
msg.append(String.format("User has superuser privilege to path %s.%n", mPath));
return new ValidationTaskResult(ValidationUtils.State.OK, getName(), msg.toString(), advice.toString());
} catch (IOException e) {
msg.append(String.format("Unable to set owner of under file system path %s: %s. ", mPath, e.getMessage()));
advice.append("Please check if Alluxio is super user on the file system. ");
return new ValidationTaskResult(ValidationUtils.State.WARNING, getName(), msg.toString(), advice.toString());
}
}
Aggregations