use of alluxio.exception.DirectoryNotEmptyException in project alluxio by Alluxio.
the class DefaultFileSystemMaster method deleteInternal.
/**
* Implements file deletion.
* <p>
* This method does not delete blocks. Instead, it returns deleted inodes so that their blocks can
* be deleted after the inode deletion journal entry has been written. We cannot delete blocks
* earlier because the inode deletion may fail, leaving us with inode containing deleted blocks.
*
* @param rpcContext the rpc context
* @param inodePath the file {@link LockedInodePath}
* @param deleteContext the method optitions
*/
@VisibleForTesting
public void deleteInternal(RpcContext rpcContext, LockedInodePath inodePath, DeleteContext deleteContext) throws FileDoesNotExistException, IOException, DirectoryNotEmptyException, InvalidPathException {
Preconditions.checkState(inodePath.getLockPattern() == LockPattern.WRITE_EDGE);
// journaled will result in an inconsistency between Alluxio and UFS.
if (!inodePath.fullPathExists()) {
return;
}
long opTimeMs = System.currentTimeMillis();
Inode inode = inodePath.getInode();
if (inode == null) {
return;
}
boolean recursive = deleteContext.getOptions().getRecursive();
if (inode.isDirectory() && !recursive && mInodeStore.hasChildren(inode.asDirectory())) {
// true
throw new DirectoryNotEmptyException(ExceptionMessage.DELETE_NONEMPTY_DIRECTORY_NONRECURSIVE, inode.getName());
}
if (mInodeTree.isRootId(inode.getId())) {
// The root cannot be deleted.
throw new InvalidPathException(ExceptionMessage.DELETE_ROOT_DIRECTORY.getMessage());
}
// Inodes for which deletion will be attempted
List<Pair<AlluxioURI, LockedInodePath>> inodesToDelete = new ArrayList<>();
// Add root of sub-tree to delete
inodesToDelete.add(new Pair<>(inodePath.getUri(), inodePath));
try (LockedInodePathList descendants = mInodeTree.getDescendants(inodePath)) {
for (LockedInodePath childPath : descendants) {
inodesToDelete.add(new Pair<>(mInodeTree.getPath(childPath.getInode()), childPath));
}
// Prepare to delete persisted inodes
UfsDeleter ufsDeleter = NoopUfsDeleter.INSTANCE;
if (!deleteContext.getOptions().getAlluxioOnly()) {
ufsDeleter = new SafeUfsDeleter(mMountTable, mInodeStore, inodesToDelete, deleteContext.getOptions().build());
}
// Inodes to delete from tree after attempting to delete from UFS
List<Pair<AlluxioURI, LockedInodePath>> revisedInodesToDelete = new ArrayList<>();
// Inodes that are not safe for recursive deletes
Set<Long> unsafeInodes = new HashSet<>();
// Alluxio URIs (and the reason for failure) which could not be deleted
List<Pair<String, String>> failedUris = new ArrayList<>();
// file, we deal with the checkpoints and blocks as well.
for (int i = inodesToDelete.size() - 1; i >= 0; i--) {
rpcContext.throwIfCancelled();
Pair<AlluxioURI, LockedInodePath> inodePairToDelete = inodesToDelete.get(i);
AlluxioURI alluxioUriToDelete = inodePairToDelete.getFirst();
Inode inodeToDelete = inodePairToDelete.getSecond().getInode();
String failureReason = null;
if (unsafeInodes.contains(inodeToDelete.getId())) {
failureReason = ExceptionMessage.DELETE_FAILED_DIR_NONEMPTY.getMessage();
} else if (inodeToDelete.isPersisted()) {
// TODO(calvin): Add tests (ALLUXIO-1831)
if (mMountTable.isMountPoint(alluxioUriToDelete)) {
mMountTable.delete(rpcContext, alluxioUriToDelete, true);
} else {
if (!deleteContext.getOptions().getAlluxioOnly()) {
try {
checkUfsMode(alluxioUriToDelete, OperationType.WRITE);
// Attempt to delete node if all children were deleted successfully
ufsDeleter.delete(alluxioUriToDelete, inodeToDelete);
} catch (AccessControlException | IOException e) {
// In case ufs is not writable, we will still attempt to delete other entries
// if any as they may be from a different mount point
LOG.warn("Failed to delete {}: {}", alluxioUriToDelete, e.toString());
failureReason = e.getMessage();
}
}
}
}
if (failureReason == null) {
if (inodeToDelete.isFile()) {
long fileId = inodeToDelete.getId();
// Remove the file from the set of files to persist.
mPersistRequests.remove(fileId);
// Cancel any ongoing jobs.
PersistJob job = mPersistJobs.get(fileId);
if (job != null) {
job.setCancelState(PersistJob.CancelState.TO_BE_CANCELED);
}
}
revisedInodesToDelete.add(new Pair<>(alluxioUriToDelete, inodePairToDelete.getSecond()));
} else {
unsafeInodes.add(inodeToDelete.getId());
// Propagate 'unsafe-ness' to parent as one of its descendants can't be deleted
unsafeInodes.add(inodeToDelete.getParentId());
failedUris.add(new Pair<>(alluxioUriToDelete.toString(), failureReason));
}
}
if (mSyncManager.isSyncPoint(inodePath.getUri())) {
mSyncManager.stopSyncAndJournal(RpcContext.NOOP, inodePath.getUri());
}
// Delete Inodes
for (Pair<AlluxioURI, LockedInodePath> delInodePair : revisedInodesToDelete) {
LockedInodePath tempInodePath = delInodePair.getSecond();
MountTable.Resolution resolution = mMountTable.resolve(tempInodePath.getUri());
mInodeTree.deleteInode(rpcContext, tempInodePath, opTimeMs);
if (deleteContext.getOptions().getAlluxioOnly()) {
Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.DELETE_FILE).inc();
}
}
if (!failedUris.isEmpty()) {
Collection<String> messages = failedUris.stream().map(pair -> String.format("%s (%s)", pair.getFirst(), pair.getSecond())).collect(Collectors.toList());
throw new FailedPreconditionException(ExceptionMessage.DELETE_FAILED_UFS.getMessage(StringUtils.join(messages, ", ")));
}
}
Metrics.PATHS_DELETED.inc(inodesToDelete.size());
}
use of alluxio.exception.DirectoryNotEmptyException in project alluxio by Alluxio.
the class InodeSyncStream method syncExistingInodeMetadata.
/**
* Sync inode metadata with the UFS state.
*
* This method expects the {@code inodePath} to already exist in the inode tree.
*/
private void syncExistingInodeMetadata(LockedInodePath inodePath, boolean skipLoad) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException, InterruptedException {
if (inodePath.getLockPattern() != LockPattern.WRITE_EDGE && !mLoadOnly) {
throw new RuntimeException(String.format("syncExistingInodeMetadata was called on d%s when only locked with %s. Load metadata" + " only was not specified.", inodePath.getUri(), inodePath.getLockPattern()));
}
// Set to true if the given inode was deleted.
boolean deletedInode = false;
// whether we need to load metadata for the current path
boolean loadMetadata = mLoadOnly;
LOG.trace("Syncing inode metadata {}", inodePath.getUri());
// The requested path already exists in Alluxio.
Inode inode = inodePath.getInode();
// initialize sync children to true if it is a listStatus call on a directory
boolean syncChildren = inode.isDirectory() && !mIsGetFileInfo && !mDescendantType.equals(DescendantType.NONE);
if (inodePath.getLockPattern() == LockPattern.WRITE_EDGE && !mLoadOnly) {
if (inode instanceof InodeFile && !inode.asFile().isCompleted()) {
// Do not sync an incomplete file, since the UFS file is expected to not exist.
return;
}
Optional<Scoped> persistingLock = mInodeLockManager.tryAcquirePersistingLock(inode.getId());
if (!persistingLock.isPresent()) {
// written.
return;
}
persistingLock.get().close();
UfsStatus cachedStatus = null;
boolean fileNotFound = false;
try {
cachedStatus = mStatusCache.getStatus(inodePath.getUri());
} catch (FileNotFoundException e) {
fileNotFound = true;
}
MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
AlluxioURI ufsUri = resolution.getUri();
try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
UnderFileSystem ufs = ufsResource.get();
String ufsFingerprint;
Fingerprint ufsFpParsed;
// When the status is not cached and it was not due to file not found, we retry
if (fileNotFound) {
ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
ufsFpParsed = Fingerprint.parse(ufsFingerprint);
} else if (cachedStatus == null) {
// TODO(david): change the interface so that getFingerprint returns a parsed fingerprint
ufsFingerprint = (String) getFromUfs(() -> ufs.getFingerprint(ufsUri.toString()));
ufsFpParsed = Fingerprint.parse(ufsFingerprint);
} else {
// When the status is cached
Pair<AccessControlList, DefaultAccessControlList> aclPair = (Pair<AccessControlList, DefaultAccessControlList>) getFromUfs(() -> ufs.getAclPair(ufsUri.toString()));
if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus);
} else {
ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus, aclPair.getFirst());
}
ufsFingerprint = ufsFpParsed.serialize();
}
boolean containsMountPoint = mMountTable.containsMountPoint(inodePath.getUri(), true);
UfsSyncUtils.SyncPlan syncPlan = UfsSyncUtils.computeSyncPlan(inode, ufsFpParsed, containsMountPoint);
if (syncPlan.toUpdateMetaData()) {
// It works by calling SetAttributeInternal on the inodePath.
if (ufsFpParsed != null && ufsFpParsed.isValid()) {
short mode = Short.parseShort(ufsFpParsed.getTag(Fingerprint.Tag.MODE));
long opTimeMs = System.currentTimeMillis();
SetAttributePOptions.Builder builder = SetAttributePOptions.newBuilder().setMode(new Mode(mode).toProto());
String owner = ufsFpParsed.getTag(Fingerprint.Tag.OWNER);
if (!owner.equals(Fingerprint.UNDERSCORE)) {
// Only set owner if not empty
builder.setOwner(owner);
}
String group = ufsFpParsed.getTag(Fingerprint.Tag.GROUP);
if (!group.equals(Fingerprint.UNDERSCORE)) {
// Only set group if not empty
builder.setGroup(group);
}
SetAttributeContext ctx = SetAttributeContext.mergeFrom(builder).setUfsFingerprint(ufsFingerprint);
mFsMaster.setAttributeSingleFile(mRpcContext, inodePath, false, opTimeMs, ctx);
}
}
if (syncPlan.toDelete()) {
deletedInode = true;
try {
// The options for deleting.
DeleteContext syncDeleteContext = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(true).setUnchecked(true));
mFsMaster.deleteInternal(mRpcContext, inodePath, syncDeleteContext);
} catch (DirectoryNotEmptyException | IOException e) {
// Should not happen, since it is an unchecked delete.
LOG.error("Unexpected error for unchecked delete.", e);
}
}
if (syncPlan.toLoadMetadata()) {
loadMetadata = true;
}
syncChildren = syncPlan.toSyncChildren();
}
}
// if DescendantType.ONE we only sync children if we are syncing root of this stream
if (mDescendantType == DescendantType.ONE) {
syncChildren = syncChildren && mRootScheme.getPath().equals(inodePath.getUri());
}
Map<String, Inode> inodeChildren = new HashMap<>();
if (syncChildren) {
// maps children name to inode
mInodeStore.getChildren(inode.asDirectory()).forEach(child -> inodeChildren.put(child.getName(), child));
// Fetch and populate children into the cache
mStatusCache.prefetchChildren(inodePath.getUri(), mMountTable);
Collection<UfsStatus> listStatus = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
// Iterate over UFS listings and process UFS children.
if (listStatus != null) {
for (UfsStatus ufsChildStatus : listStatus) {
if (!inodeChildren.containsKey(ufsChildStatus.getName()) && !PathUtils.isTemporaryFileName(ufsChildStatus.getName())) {
// Ufs child exists, but Alluxio child does not. Must load metadata.
loadMetadata = true;
break;
}
}
}
}
// locked path
if (deletedInode) {
inodePath.removeLastInode();
}
// load metadata if necessary.
if (loadMetadata && !skipLoad) {
loadMetadataForPath(inodePath);
}
if (syncChildren) {
// Iterate over Alluxio children and process persisted children.
mInodeStore.getChildren(inode.asDirectory()).forEach(childInode -> {
// its children.
if (mLoadOnly && inodeChildren.containsKey(childInode.getName()) && childInode.isFile()) {
return;
}
// If we're performing a recursive sync, add each child of our current Inode to the queue
AlluxioURI child = inodePath.getUri().joinUnsafe(childInode.getName());
mPendingPaths.add(child);
// This asynchronously schedules a job to pre-fetch the statuses into the cache.
if (childInode.isDirectory() && mDescendantType == DescendantType.ALL) {
mStatusCache.prefetchChildren(child, mMountTable);
}
});
}
}
use of alluxio.exception.DirectoryNotEmptyException in project alluxio by Alluxio.
the class S3RestServiceHandler method postBucket.
/**
* Currently implements the DeleteObjects request type if the query parameter "delete" exists.
*
* @param bucket the bucket name
* @param delete the delete query parameter. Existence indicates to run the DeleteObjects impl
* @param contentLength body content length
* @param is the input stream to read the request
*
* @return a {@link DeleteObjectsResult} if this was a DeleteObjects request
*/
@POST
@Path(BUCKET_PARAM)
public Response postBucket(@PathParam("bucket") final String bucket, @QueryParam("delete") String delete, @HeaderParam("Content-Length") int contentLength, final InputStream is) {
return S3RestUtils.call(bucket, () -> {
if (delete != null) {
try {
DeleteObjectsRequest request = new XmlMapper().readerFor(DeleteObjectsRequest.class).readValue(is);
List<DeleteObjectsRequest.DeleteObject> objs = request.getToDelete();
List<DeleteObjectsResult.DeletedObject> success = new ArrayList<>();
List<DeleteObjectsResult.ErrorObject> errored = new ArrayList<>();
objs.sort(Comparator.comparingInt(x -> -1 * x.getKey().length()));
objs.forEach(obj -> {
try {
AlluxioURI uri = new AlluxioURI(AlluxioURI.SEPARATOR + bucket).join(AlluxioURI.SEPARATOR + obj.getKey());
DeletePOptions options = DeletePOptions.newBuilder().build();
mFileSystem.delete(uri, options);
DeleteObjectsResult.DeletedObject del = new DeleteObjectsResult.DeletedObject();
del.setKey(obj.getKey());
success.add(del);
} catch (FileDoesNotExistException | DirectoryNotEmptyException e) {
/*
FDNE - delete on FDNE should be counted as a success, as there's nothing to do
DNE - s3 has no concept dirs - if it _is_ a dir, nothing to delete.
*/
DeleteObjectsResult.DeletedObject del = new DeleteObjectsResult.DeletedObject();
del.setKey(obj.getKey());
success.add(del);
} catch (IOException | AlluxioException e) {
DeleteObjectsResult.ErrorObject err = new DeleteObjectsResult.ErrorObject();
err.setKey(obj.getKey());
err.setMessage(e.getMessage());
errored.add(err);
}
});
DeleteObjectsResult result = new DeleteObjectsResult();
if (!request.getQuiet()) {
result.setDeleted(success);
}
result.setErrored(errored);
return result;
} catch (IOException e) {
LOG.debug("Failed to parse DeleteObjects request:", e);
return Response.Status.BAD_REQUEST;
}
} else {
return Response.Status.OK;
}
});
}
use of alluxio.exception.DirectoryNotEmptyException in project alluxio by Alluxio.
the class S3RestServiceHandler method deleteObject.
private void deleteObject(FileSystem fs, String bucket, String object) throws S3Exception {
String bucketPath = S3RestUtils.parsePath(AlluxioURI.SEPARATOR + bucket);
// Delete the object.
String objectPath = bucketPath + AlluxioURI.SEPARATOR + object;
DeletePOptions options = DeletePOptions.newBuilder().setAlluxioOnly(ServerConfiguration.get(PropertyKey.PROXY_S3_DELETE_TYPE).equals(Constants.S3_DELETE_IN_ALLUXIO_ONLY)).build();
try {
fs.delete(new AlluxioURI(objectPath), options);
} catch (FileDoesNotExistException | DirectoryNotEmptyException e) {
// intentionally do nothing, this is ok. It should result in a 204 error
// This is the same response behavior as AWS's S3.
} catch (Exception e) {
throw S3RestUtils.toObjectS3Exception(e, objectPath);
}
}
use of alluxio.exception.DirectoryNotEmptyException in project alluxio by Alluxio.
the class FileSystemMasterIntegrationTest method deleteDirectoryWithDirectoriesTest2.
@Test
public void deleteDirectoryWithDirectoriesTest2() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryContext.defaults());
mFsMaster.createDirectory(new AlluxioURI("/testFolder/testFolder2"), CreateDirectoryContext.defaults());
long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileContext.defaults()).getFileId();
long fileId2 = mFsMaster.createFile(new AlluxioURI("/testFolder/testFolder2/testFile2"), CreateFileContext.defaults()).getFileId();
Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile")));
Assert.assertEquals(fileId2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2")));
try {
mFsMaster.delete(new AlluxioURI("/testFolder/testFolder2"), DeleteContext.defaults());
Assert.fail("Deleting a nonempty directory nonrecursively should fail");
} catch (DirectoryNotEmptyException e) {
Assert.assertEquals(ExceptionMessage.DELETE_NONEMPTY_DIRECTORY_NONRECURSIVE.getMessage("testFolder2"), e.getMessage());
}
Assert.assertEquals(fileId, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile")));
Assert.assertEquals(fileId2, mFsMaster.getFileId(new AlluxioURI("/testFolder/testFolder2/testFile2")));
}
Aggregations