Search in sources :

Example 1 with Journaled

use of alluxio.master.journal.Journaled in project alluxio by Alluxio.

the class DefaultFileSystemMaster method deleteInternal.

/**
 * Implements file deletion.
 * <p>
 * This method does not delete blocks. Instead, it returns deleted inodes so that their blocks can
 * be deleted after the inode deletion journal entry has been written. We cannot delete blocks
 * earlier because the inode deletion may fail, leaving us with inode containing deleted blocks.
 *
 * @param rpcContext the rpc context
 * @param inodePath the file {@link LockedInodePath}
 * @param deleteContext the method optitions
 */
@VisibleForTesting
public void deleteInternal(RpcContext rpcContext, LockedInodePath inodePath, DeleteContext deleteContext) throws FileDoesNotExistException, IOException, DirectoryNotEmptyException, InvalidPathException {
    Preconditions.checkState(inodePath.getLockPattern() == LockPattern.WRITE_EDGE);
    // journaled will result in an inconsistency between Alluxio and UFS.
    if (!inodePath.fullPathExists()) {
        return;
    }
    long opTimeMs = System.currentTimeMillis();
    Inode inode = inodePath.getInode();
    if (inode == null) {
        return;
    }
    boolean recursive = deleteContext.getOptions().getRecursive();
    if (inode.isDirectory() && !recursive && mInodeStore.hasChildren(inode.asDirectory())) {
        // true
        throw new DirectoryNotEmptyException(ExceptionMessage.DELETE_NONEMPTY_DIRECTORY_NONRECURSIVE, inode.getName());
    }
    if (mInodeTree.isRootId(inode.getId())) {
        // The root cannot be deleted.
        throw new InvalidPathException(ExceptionMessage.DELETE_ROOT_DIRECTORY.getMessage());
    }
    // Inodes for which deletion will be attempted
    List<Pair<AlluxioURI, LockedInodePath>> inodesToDelete = new ArrayList<>();
    // Add root of sub-tree to delete
    inodesToDelete.add(new Pair<>(inodePath.getUri(), inodePath));
    try (LockedInodePathList descendants = mInodeTree.getDescendants(inodePath)) {
        for (LockedInodePath childPath : descendants) {
            inodesToDelete.add(new Pair<>(mInodeTree.getPath(childPath.getInode()), childPath));
        }
        // Prepare to delete persisted inodes
        UfsDeleter ufsDeleter = NoopUfsDeleter.INSTANCE;
        if (!deleteContext.getOptions().getAlluxioOnly()) {
            ufsDeleter = new SafeUfsDeleter(mMountTable, mInodeStore, inodesToDelete, deleteContext.getOptions().build());
        }
        // Inodes to delete from tree after attempting to delete from UFS
        List<Pair<AlluxioURI, LockedInodePath>> revisedInodesToDelete = new ArrayList<>();
        // Inodes that are not safe for recursive deletes
        Set<Long> unsafeInodes = new HashSet<>();
        // Alluxio URIs (and the reason for failure) which could not be deleted
        List<Pair<String, String>> failedUris = new ArrayList<>();
        // file, we deal with the checkpoints and blocks as well.
        for (int i = inodesToDelete.size() - 1; i >= 0; i--) {
            rpcContext.throwIfCancelled();
            Pair<AlluxioURI, LockedInodePath> inodePairToDelete = inodesToDelete.get(i);
            AlluxioURI alluxioUriToDelete = inodePairToDelete.getFirst();
            Inode inodeToDelete = inodePairToDelete.getSecond().getInode();
            String failureReason = null;
            if (unsafeInodes.contains(inodeToDelete.getId())) {
                failureReason = ExceptionMessage.DELETE_FAILED_DIR_NONEMPTY.getMessage();
            } else if (inodeToDelete.isPersisted()) {
                // TODO(calvin): Add tests (ALLUXIO-1831)
                if (mMountTable.isMountPoint(alluxioUriToDelete)) {
                    mMountTable.delete(rpcContext, alluxioUriToDelete, true);
                } else {
                    if (!deleteContext.getOptions().getAlluxioOnly()) {
                        try {
                            checkUfsMode(alluxioUriToDelete, OperationType.WRITE);
                            // Attempt to delete node if all children were deleted successfully
                            ufsDeleter.delete(alluxioUriToDelete, inodeToDelete);
                        } catch (AccessControlException | IOException e) {
                            // In case ufs is not writable, we will still attempt to delete other entries
                            // if any as they may be from a different mount point
                            LOG.warn("Failed to delete {}: {}", alluxioUriToDelete, e.toString());
                            failureReason = e.getMessage();
                        }
                    }
                }
            }
            if (failureReason == null) {
                if (inodeToDelete.isFile()) {
                    long fileId = inodeToDelete.getId();
                    // Remove the file from the set of files to persist.
                    mPersistRequests.remove(fileId);
                    // Cancel any ongoing jobs.
                    PersistJob job = mPersistJobs.get(fileId);
                    if (job != null) {
                        job.setCancelState(PersistJob.CancelState.TO_BE_CANCELED);
                    }
                }
                revisedInodesToDelete.add(new Pair<>(alluxioUriToDelete, inodePairToDelete.getSecond()));
            } else {
                unsafeInodes.add(inodeToDelete.getId());
                // Propagate 'unsafe-ness' to parent as one of its descendants can't be deleted
                unsafeInodes.add(inodeToDelete.getParentId());
                failedUris.add(new Pair<>(alluxioUriToDelete.toString(), failureReason));
            }
        }
        if (mSyncManager.isSyncPoint(inodePath.getUri())) {
            mSyncManager.stopSyncAndJournal(RpcContext.NOOP, inodePath.getUri());
        }
        // Delete Inodes
        for (Pair<AlluxioURI, LockedInodePath> delInodePair : revisedInodesToDelete) {
            LockedInodePath tempInodePath = delInodePair.getSecond();
            MountTable.Resolution resolution = mMountTable.resolve(tempInodePath.getUri());
            mInodeTree.deleteInode(rpcContext, tempInodePath, opTimeMs);
            if (deleteContext.getOptions().getAlluxioOnly()) {
                Metrics.getUfsOpsSavedCounter(resolution.getUfsMountPointUri(), Metrics.UFSOps.DELETE_FILE).inc();
            }
        }
        if (!failedUris.isEmpty()) {
            Collection<String> messages = failedUris.stream().map(pair -> String.format("%s (%s)", pair.getFirst(), pair.getSecond())).collect(Collectors.toList());
            throw new FailedPreconditionException(ExceptionMessage.DELETE_FAILED_UFS.getMessage(StringUtils.join(messages, ", ")));
        }
    }
    Metrics.PATHS_DELETED.inc(inodesToDelete.size());
}
Also used : SystemClock(alluxio.clock.SystemClock) OK(alluxio.master.file.InodeSyncStream.SyncStatus.OK) Server(alluxio.Server) PropertyKey(alluxio.conf.PropertyKey) StringUtils(org.apache.commons.lang3.StringUtils) CloseableResource(alluxio.resource.CloseableResource) Map(java.util.Map) LockedInodePathList(alluxio.master.file.meta.LockedInodePathList) WorkerHeartbeatContext(alluxio.master.file.contexts.WorkerHeartbeatContext) InodeDirectory(alluxio.master.file.meta.InodeDirectory) ClientContext(alluxio.ClientContext) ReadOnlyInodeStore(alluxio.master.metastore.ReadOnlyInodeStore) UfsSyncPathCache(alluxio.master.file.meta.UfsSyncPathCache) ConnectionFailedException(alluxio.exception.ConnectionFailedException) ProtobufUtils(alluxio.master.ProtobufUtils) UpdateInodeFileEntry(alluxio.proto.journal.File.UpdateInodeFileEntry) Stream(java.util.stream.Stream) InodeTree(alluxio.master.file.meta.InodeTree) OperationContext(alluxio.master.file.contexts.OperationContext) SetAclAction(alluxio.grpc.SetAclAction) GetStatusPOptions(alluxio.grpc.GetStatusPOptions) InternalOperationContext(alluxio.master.file.contexts.InternalOperationContext) InodeFile(alluxio.master.file.meta.InodeFile) PersistConfig(alluxio.job.plan.persist.PersistConfig) AuthType(alluxio.security.authentication.AuthType) Supplier(java.util.function.Supplier) UnderFileSystemUtils(alluxio.util.UnderFileSystemUtils) LockedInodePath(alluxio.master.file.meta.LockedInodePath) GrpcUtils(alluxio.grpc.GrpcUtils) MountInfo(alluxio.master.file.meta.options.MountInfo) AlluxioURI(alluxio.AlluxioURI) MountPOptions(alluxio.grpc.MountPOptions) MetricsSystem(alluxio.metrics.MetricsSystem) UpdateInodeEntry(alluxio.proto.journal.File.UpdateInodeEntry) RetryPolicy(alluxio.retry.RetryPolicy) PersistFile(alluxio.wire.PersistFile) IdUtils(alluxio.util.IdUtils) PersistenceState(alluxio.master.file.meta.PersistenceState) IOException(java.io.IOException) UfsAbsentPathCache(alluxio.master.file.meta.UfsAbsentPathCache) TreeMap(java.util.TreeMap) UfsStatus(alluxio.underfs.UfsStatus) UfsFileStatus(alluxio.underfs.UfsFileStatus) Preconditions(com.google.common.base.Preconditions) FreeContext(alluxio.master.file.contexts.FreeContext) AclEntry(alluxio.security.authorization.AclEntry) Reconfigurable(alluxio.conf.Reconfigurable) MountTable(alluxio.master.file.meta.MountTable) CommonUtils(alluxio.util.CommonUtils) InodeLockManager(alluxio.master.file.meta.InodeLockManager) InodePathPair(alluxio.master.file.meta.InodePathPair) FailedPreconditionException(alluxio.exception.status.FailedPreconditionException) Fingerprint(alluxio.underfs.Fingerprint) AclEntryType(alluxio.security.authorization.AclEntryType) DeleteContext(alluxio.master.file.contexts.DeleteContext) LoadMetadataContext(alluxio.master.file.contexts.LoadMetadataContext) RenameEntry(alluxio.proto.journal.File.RenameEntry) HeartbeatThread(alluxio.heartbeat.HeartbeatThread) InvalidPathException(alluxio.exception.InvalidPathException) ListStatusContext(alluxio.master.file.contexts.ListStatusContext) MetricKey(alluxio.metrics.MetricKey) SyncPointInfo(alluxio.wire.SyncPointInfo) FileSystemMasterCommonPOptions(alluxio.grpc.FileSystemMasterCommonPOptions) Journaled(alluxio.master.journal.Journaled) ImmutableSet(com.google.common.collect.ImmutableSet) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) CheckpointName(alluxio.master.journal.checkpoint.CheckpointName) Collectors(java.util.stream.Collectors) MkdirsOptions(alluxio.underfs.options.MkdirsOptions) InodeStore(alluxio.master.metastore.InodeStore) SecurityUtils(alluxio.util.SecurityUtils) CoreMasterContext(alluxio.master.CoreMasterContext) CommandType(alluxio.wire.CommandType) SetAttributeContext(alluxio.master.file.contexts.SetAttributeContext) FileAlreadyCompletedException(alluxio.exception.FileAlreadyCompletedException) Function(java.util.function.Function) Stack(java.util.Stack) MetricInfo(alluxio.metrics.MetricInfo) ExistsContext(alluxio.master.file.contexts.ExistsContext) HashSet(java.util.HashSet) Constants(alluxio.Constants) FileSystemMasterView(alluxio.master.file.meta.FileSystemMasterView) NOT_NEEDED(alluxio.master.file.InodeSyncStream.SyncStatus.NOT_NEEDED) ExecutorService(java.util.concurrent.ExecutorService) AuthenticatedClientUser(alluxio.security.authentication.AuthenticatedClientUser) SetAclEntry(alluxio.proto.journal.File.SetAclEntry) Logger(org.slf4j.Logger) Pair(alluxio.collections.Pair) AsyncUserAccessAuditLogWriter(alluxio.master.audit.AsyncUserAccessAuditLogWriter) NotFoundException(alluxio.exception.status.NotFoundException) LockResource(alluxio.resource.LockResource) UnexpectedAlluxioException(alluxio.exception.UnexpectedAlluxioException) CallTracker(alluxio.master.file.contexts.CallTracker) UfsBlockLocationCache(alluxio.master.file.meta.UfsBlockLocationCache) CreateDirectoryContext(alluxio.master.file.contexts.CreateDirectoryContext) CreateFileContext(alluxio.master.file.contexts.CreateFileContext) Arrays(java.util.Arrays) BlockInfo(alluxio.wire.BlockInfo) PersistCommandOptions(alluxio.wire.PersistCommandOptions) GrpcService(alluxio.grpc.GrpcService) TimeSeries(alluxio.metrics.TimeSeries) WorkerInfo(alluxio.wire.WorkerInfo) DelegatingJournaled(alluxio.master.journal.DelegatingJournaled) PreconditionMessage(alluxio.exception.PreconditionMessage) AuditContext(alluxio.master.audit.AuditContext) DescendantType(alluxio.file.options.DescendantType) ActiveSyncManager(alluxio.master.file.activesync.ActiveSyncManager) Set(java.util.Set) AlluxioException(alluxio.exception.AlluxioException) GetStatusContext(alluxio.master.file.contexts.GetStatusContext) UnderFileSystem(alluxio.underfs.UnderFileSystem) JobInfo(alluxio.job.wire.JobInfo) ServiceType(alluxio.grpc.ServiceType) Iterables(com.google.common.collect.Iterables) CountingRetry(alluxio.retry.CountingRetry) UnderFileSystemConfiguration(alluxio.underfs.UnderFileSystemConfiguration) Callable(java.util.concurrent.Callable) InodeDirectoryView(alluxio.master.file.meta.InodeDirectoryView) Mode(alluxio.security.authorization.Mode) Metric(alluxio.metrics.Metric) ArrayList(java.util.ArrayList) ReconfigurableRegistry(alluxio.conf.ReconfigurableRegistry) BlockInfoException(alluxio.exception.BlockInfoException) InodeDirectoryIdGenerator(alluxio.master.file.meta.InodeDirectoryIdGenerator) Builder(alluxio.proto.journal.File.UpdateInodeFileEntry.Builder) Nullable(javax.annotation.Nullable) LoadDescendantPType(alluxio.grpc.LoadDescendantPType) MountContext(alluxio.master.file.contexts.MountContext) MetricRegistry(com.codahale.metrics.MetricRegistry) FileAlreadyExistsException(alluxio.exception.FileAlreadyExistsException) RenameContext(alluxio.master.file.contexts.RenameContext) FileSystemCommandOptions(alluxio.wire.FileSystemCommandOptions) JobMasterClientPool(alluxio.client.job.JobMasterClientPool) FileSystemCommand(alluxio.wire.FileSystemCommand) ExecutorServiceFactory(alluxio.util.executor.ExecutorServiceFactory) UfsInfo(alluxio.wire.UfsInfo) JournaledGroup(alluxio.master.journal.JournaledGroup) ExecutorServiceFactories(alluxio.util.executor.ExecutorServiceFactories) NotThreadSafe(javax.annotation.concurrent.NotThreadSafe) ServerInterceptors(io.grpc.ServerInterceptors) MasterUfsManager(alluxio.underfs.MasterUfsManager) ScheduleAsyncPersistenceContext(alluxio.master.file.contexts.ScheduleAsyncPersistenceContext) LoggerFactory(org.slf4j.LoggerFactory) FileBlockInfo(alluxio.wire.FileBlockInfo) NewBlockEntry(alluxio.proto.journal.File.NewBlockEntry) LogUtils(alluxio.util.LogUtils) ResourceExhaustedException(alluxio.exception.status.ResourceExhaustedException) Counter(com.codahale.metrics.Counter) JobMasterClient(alluxio.client.job.JobMasterClient) CheckConsistencyContext(alluxio.master.file.contexts.CheckConsistencyContext) InvalidArgumentException(alluxio.exception.status.InvalidArgumentException) LockPattern(alluxio.master.file.meta.InodeTree.LockPattern) TtlAction(alluxio.grpc.TtlAction) ServerConfiguration(alluxio.conf.ServerConfiguration) ImmutableMap(com.google.common.collect.ImmutableMap) CompleteFileContext(alluxio.master.file.contexts.CompleteFileContext) TimeSeriesStore(alluxio.master.metrics.TimeSeriesStore) PermissionDeniedException(alluxio.exception.status.PermissionDeniedException) LoadMetadataPOptions(alluxio.grpc.LoadMetadataPOptions) Streams(com.google.common.collect.Streams) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Sets(com.google.common.collect.Sets) FileNotFoundException(java.io.FileNotFoundException) DelegatingReadOnlyInodeStore(alluxio.master.metastore.DelegatingReadOnlyInodeStore) AccessControlException(alluxio.exception.AccessControlException) File(alluxio.proto.journal.File) List(java.util.List) InvalidFileSizeException(alluxio.exception.InvalidFileSizeException) FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) LoadMetadataPType(alluxio.grpc.LoadMetadataPType) ModeUtils(alluxio.util.ModeUtils) Gauge(com.codahale.metrics.Gauge) UFS_OP_SAVED_PREFIX(alluxio.metrics.MetricInfo.UFS_OP_SAVED_PREFIX) UfsManager(alluxio.underfs.UfsManager) JournalContext(alluxio.master.journal.JournalContext) SortedMap(java.util.SortedMap) UnavailableException(alluxio.exception.status.UnavailableException) MountPointInfo(alluxio.wire.MountPointInfo) JobMasterClientContext(alluxio.worker.job.JobMasterClientContext) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) HashMap(java.util.HashMap) SetAttributePOptions(alluxio.grpc.SetAttributePOptions) BlockId(alluxio.master.block.BlockId) BlockMaster(alluxio.master.block.BlockMaster) CheckAccessContext(alluxio.master.file.contexts.CheckAccessContext) JournalEntry(alluxio.proto.journal.Journal.JournalEntry) DeletePOptions(alluxio.grpc.DeletePOptions) PathUtils(alluxio.util.io.PathUtils) DirectoryNotEmptyException(alluxio.exception.DirectoryNotEmptyException) ClientIpAddressInjector(alluxio.security.authentication.ClientIpAddressInjector) PrefixList(alluxio.collections.PrefixList) SetAclContext(alluxio.master.file.contexts.SetAclContext) CoreMaster(alluxio.master.CoreMaster) HeartbeatContext(alluxio.heartbeat.HeartbeatContext) Iterator(java.util.Iterator) ExceptionMessage(alluxio.exception.ExceptionMessage) ThreadFactoryUtils(alluxio.util.ThreadFactoryUtils) ProtoUtils(alluxio.util.proto.ProtoUtils) TimeUnit(java.util.concurrent.TimeUnit) BlockLocation(alluxio.wire.BlockLocation) FAILED(alluxio.master.file.InodeSyncStream.SyncStatus.FAILED) LockingScheme(alluxio.master.file.meta.LockingScheme) FileInfo(alluxio.wire.FileInfo) VisibleForTesting(com.google.common.annotations.VisibleForTesting) UfsMode(alluxio.underfs.UfsMode) Collections(java.util.Collections) Inode(alluxio.master.file.meta.Inode) ArrayList(java.util.ArrayList) DirectoryNotEmptyException(alluxio.exception.DirectoryNotEmptyException) LockedInodePathList(alluxio.master.file.meta.LockedInodePathList) MountTable(alluxio.master.file.meta.MountTable) InvalidPathException(alluxio.exception.InvalidPathException) InodePathPair(alluxio.master.file.meta.InodePathPair) Pair(alluxio.collections.Pair) HashSet(java.util.HashSet) Fingerprint(alluxio.underfs.Fingerprint) LockedInodePath(alluxio.master.file.meta.LockedInodePath) Inode(alluxio.master.file.meta.Inode) FailedPreconditionException(alluxio.exception.status.FailedPreconditionException) AlluxioURI(alluxio.AlluxioURI) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 2 with Journaled

use of alluxio.master.journal.Journaled in project alluxio by Alluxio.

the class BufferedJournalApplier method applyToMaster.

/**
 * Applies the entry to master and updates last applied sequence.
 * Calls to it should be serialized.
 */
private void applyToMaster(Journal.JournalEntry entry) {
    String masterName;
    try {
        masterName = JournalEntryAssociation.getMasterForEntry(entry);
    } catch (Exception t) {
        ProcessUtils.fatalError(LOG, t, "Unrecognized journal entry: %s", entry);
        throw new IllegalStateException();
    }
    try {
        Journaled master = mJournals.get(masterName).getStateMachine();
        LOG.trace("Applying entry to master {}: {} ", masterName, entry);
        master.processJournalEntry(entry);
        JournalUtils.sinkAppend(mJournalSinks, entry);
    } catch (Exception t) {
        JournalUtils.handleJournalReplayFailure(LOG, t, "Failed to apply journal entry to master %s. Entry: %s", masterName, entry);
    }
    // Store last applied sequence.
    mLastAppliedSequence = entry.getSequenceNumber();
}
Also used : Journaled(alluxio.master.journal.Journaled) IOException(java.io.IOException)

Aggregations

Journaled (alluxio.master.journal.Journaled)2 IOException (java.io.IOException)2 AlluxioURI (alluxio.AlluxioURI)1 ClientContext (alluxio.ClientContext)1 Constants (alluxio.Constants)1 Server (alluxio.Server)1 JobMasterClient (alluxio.client.job.JobMasterClient)1 JobMasterClientPool (alluxio.client.job.JobMasterClientPool)1 SystemClock (alluxio.clock.SystemClock)1 Pair (alluxio.collections.Pair)1 PrefixList (alluxio.collections.PrefixList)1 PropertyKey (alluxio.conf.PropertyKey)1 Reconfigurable (alluxio.conf.Reconfigurable)1 ReconfigurableRegistry (alluxio.conf.ReconfigurableRegistry)1 ServerConfiguration (alluxio.conf.ServerConfiguration)1 AccessControlException (alluxio.exception.AccessControlException)1 AlluxioException (alluxio.exception.AlluxioException)1 BlockInfoException (alluxio.exception.BlockInfoException)1 ConnectionFailedException (alluxio.exception.ConnectionFailedException)1 DirectoryNotEmptyException (alluxio.exception.DirectoryNotEmptyException)1