Examples with LockedInodePath alluxio.master.file.meta.LockedInodePath used on opensource projects

Example 76 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getFileIdInternal.

private long getFileIdInternal(AlluxioURI path, boolean checkPermission) throws AccessControlException, UnavailableException {
    try (RpcContext rpcContext = createRpcContext()) {
        /*
      In order to prevent locking twice on RPCs where metadata does _not_ need to be loaded, we use
      a two-step scheme as an optimization to prevent the extra lock. loadMetadataIfNotExists
      requires a lock on the tree to determine if the path should be loaded before executing. To
      prevent the extra lock, we execute the RPC as normal and use a conditional check in the
      main body of the function to determine whether control flow should be shifted out of the
      RPC logic and back to the loadMetadataIfNotExists function.

      If loadMetadataIfNotExists runs, then the next pass into the main logic body should
      continue as normal. This may present a slight decrease in performance for newly-loaded
      metadata, but it is better than affecting the most common case where metadata is not being
      loaded.
       */
        LoadMetadataContext lmCtx = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setCreateAncestors(true));
        boolean run = true;
        boolean loadMetadata = false;
        while (run) {
            run = false;
            if (loadMetadata) {
                loadMetadataIfNotExist(rpcContext, path, lmCtx, false);
            }
            try (LockedInodePath inodePath = mInodeTree.lockInodePath(path, LockPattern.READ)) {
                if (checkPermission) {
                    mPermissionChecker.checkPermission(Mode.Bits.READ, inodePath);
                }
                if (!loadMetadata && shouldLoadMetadataIfNotExists(inodePath, lmCtx)) {
                    loadMetadata = true;
                    run = true;
                    continue;
                }
                mInodeTree.ensureFullInodePath(inodePath);
                return inodePath.getInode().getId();
            } catch (InvalidPathException | FileDoesNotExistException e) {
                return IdUtils.INVALID_FILE_ID;
            }
        }
    } catch (InvalidPathException e) {
        return IdUtils.INVALID_FILE_ID;
    }
    return IdUtils.INVALID_FILE_ID;
}

Example 77 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class InodeSyncStream method sync.

/**
 * Sync the metadata according the root path the stream was created with.
 *
 * @return SyncStatus object
 */
public SyncStatus sync() throws AccessControlException, InvalidPathException {
    // The high-level process for the syncing is:
    // 1. Given an Alluxio path, determine if it is not consistent with the corresponding UFS path.
    // this means the UFS path does not exist, or has metadata which differs from Alluxio
    // 2. If only the metadata changed, update the inode with the new metadata
    // 3. If the path does not exist in the UFS, delete the inode in Alluxio
    // 4. If not deleted, load metadata from the UFS
    // 5. If a recursive sync, add children inodes to sync queue
    int syncPathCount = 0;
    int failedSyncPathCount = 0;
    // stop syncing when we've processed this many paths. -1 for infinite
    int stopNum = -1;
    if (!mRootScheme.shouldSync() && !mForceSync) {
        return SyncStatus.NOT_NEEDED;
    }
    Instant start = Instant.now();
    try (LockedInodePath path = mInodeTree.lockInodePath(mRootScheme)) {
        if (mAuditContext != null && mAuditContextSrcInodeFunc != null) {
            mAuditContext.setSrcInode(mAuditContextSrcInodeFunc.apply(path));
        }
        syncInodeMetadata(path);
        syncPathCount++;
        if (mDescendantType == DescendantType.ONE) {
            // If descendantType is ONE, then we shouldn't process any more paths except for those
            // currently in the queue
            stopNum = mPendingPaths.size();
        }
        // process the sync result for the original path
        try {
            path.traverse();
        } catch (InvalidPathException e) {
            throw new RuntimeException(e);
        }
    } catch (BlockInfoException | FileAlreadyCompletedException | FileDoesNotExistException | InterruptedException | InvalidFileSizeException | IOException e) {
        LogUtils.warnWithException(LOG, "Failed to sync metadata on root path {}", toString(), e);
        failedSyncPathCount++;
    } finally {
        // regardless of the outcome, remove the UfsStatus for this path from the cache
        mStatusCache.remove(mRootScheme.getPath());
    }
    // Process any children after the root.
    while (!mPendingPaths.isEmpty() || !mSyncPathJobs.isEmpty()) {
        if (Thread.currentThread().isInterrupted()) {
            LOG.warn("Metadata syncing was interrupted before completion; {}", toString());
            break;
        }
        if (mRpcContext.isCancelled()) {
            LOG.warn("Metadata syncing was cancelled before completion; {}", toString());
            break;
        }
        // successfully
        while (true) {
            Future<Boolean> job = mSyncPathJobs.peek();
            if (job == null || !job.isDone()) {
                break;
            }
            // remove the job because we know it is done.
            if (mSyncPathJobs.poll() != job) {
                throw new ConcurrentModificationException("Head of queue modified while executing");
            }
            try {
                // This shouldn't block because we checked job.isDone() earlier
                if (job.get()) {
                    syncPathCount++;
                } else {
                    failedSyncPathCount++;
                }
            } catch (InterruptedException | ExecutionException e) {
                failedSyncPathCount++;
                LogUtils.warnWithException(LOG, "metadata sync failed while polling for finished paths; {}", toString(), e);
                if (e instanceof InterruptedException) {
                    Thread.currentThread().interrupt();
                    break;
                }
            }
        }
        // When using descendant type of ONE, we need to stop prematurely.
        if (stopNum != -1 && (syncPathCount + failedSyncPathCount) > stopNum) {
            break;
        }
        // We can submit up to ( max_concurrency - <jobs queue size>) jobs back into the queue
        int submissions = mConcurrencyLevel - mSyncPathJobs.size();
        for (int i = 0; i < submissions; i++) {
            AlluxioURI path = mPendingPaths.poll();
            if (path == null) {
                // no paths left to sync
                break;
            }
            Future<Boolean> job = mMetadataSyncService.submit(() -> processSyncPath(path));
            mSyncPathJobs.offer(job);
        }
        // After submitting all jobs wait for the job at the head of the queue to finish.
        Future<Boolean> oldestJob = mSyncPathJobs.peek();
        if (oldestJob == null) {
            // There might not be any jobs, restart the loop.
            continue;
        }
        try {
            // block until the oldest job finished.
            oldestJob.get();
        } catch (InterruptedException | ExecutionException e) {
            LogUtils.warnWithException(LOG, "Exception while waiting for oldest metadata sync job to finish: {}", toString(), e);
            if (e instanceof InterruptedException) {
                Thread.currentThread().interrupt();
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        Instant end = Instant.now();
        Duration elapsedTime = Duration.between(start, end);
        LOG.debug("synced {} paths ({} success, {} failed) in {} ms on {}", syncPathCount + failedSyncPathCount, syncPathCount, failedSyncPathCount, elapsedTime.toMillis(), mRootScheme);
    }
    boolean success = syncPathCount > 0;
    if (ServerConfiguration.getBoolean(PropertyKey.MASTER_METADATA_SYNC_REPORT_FAILURE)) {
        // There should not be any failed or outstanding jobs
        success = (failedSyncPathCount == 0) && mSyncPathJobs.isEmpty() && mPendingPaths.isEmpty();
    }
    if (success) {
        // update the sync path cache for the root of the sync
        // TODO(gpang): Do we need special handling for failures and thread interrupts?
        mUfsSyncPathCache.notifySyncedPath(mRootScheme.getPath().getPath(), mDescendantType);
    }
    mStatusCache.cancelAllPrefetch();
    mSyncPathJobs.forEach(f -> f.cancel(true));
    return success ? SyncStatus.OK : SyncStatus.FAILED;
}

Example 78 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class InodeSyncStream method loadMetadata.

/**
 * This method creates inodes containing the metadata from the UFS. The {@link UfsStatus} object
 * must be set in the {@link LoadMetadataContext} in order to successfully create the inodes.
 */
private void loadMetadata(LockedInodePath inodePath, LoadMetadataContext context) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException {
    AlluxioURI path = inodePath.getUri();
    MountTable.Resolution resolution = mMountTable.resolve(path);
    int failedSync = 0;
    try {
        if (context.getUfsStatus() == null) {
            // uri does not exist in ufs
            Inode inode = inodePath.getInode();
            if (inode.isFile()) {
                throw new IllegalArgumentException(String.format("load metadata cannot be called on a file if no ufs " + "status is present in the context. %s", inodePath.getUri()));
            }
            mInodeTree.setDirectChildrenLoaded(mRpcContext, inode.asDirectory());
            return;
        }
        if (context.getUfsStatus().isFile()) {
            loadFileMetadataInternal(mRpcContext, inodePath, resolution, context, mFsMaster);
        } else {
            loadDirectoryMetadata(mRpcContext, inodePath, context, mMountTable, mFsMaster);
            // now load all children if required
            LoadDescendantPType type = context.getOptions().getLoadDescendantType();
            if (type != LoadDescendantPType.NONE) {
                Collection<UfsStatus> children = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
                if (children == null) {
                    LOG.debug("fetching children for {} returned null", inodePath.getUri());
                    return;
                }
                for (UfsStatus childStatus : children) {
                    if (PathUtils.isTemporaryFileName(childStatus.getName())) {
                        continue;
                    }
                    AlluxioURI childURI = new AlluxioURI(PathUtils.concatPath(inodePath.getUri(), childStatus.getName()));
                    if (mInodeTree.inodePathExists(childURI) && (childStatus.isFile() || context.getOptions().getLoadDescendantType() != LoadDescendantPType.ALL)) {
                        // loading all descendants.
                        continue;
                    }
                    LoadMetadataContext loadMetadataContext = LoadMetadataContext.mergeFrom(LoadMetadataPOptions.newBuilder().setLoadDescendantType(LoadDescendantPType.NONE).setCommonOptions(NO_TTL_OPTION).setCreateAncestors(false)).setUfsStatus(childStatus);
                    try (LockedInodePath descendant = inodePath.lockDescendant(inodePath.getUri().joinUnsafe(childStatus.getName()), LockPattern.READ)) {
                        loadMetadata(descendant, loadMetadataContext);
                    } catch (FileNotFoundException e) {
                        LOG.debug("Failed to loadMetadata because file is not in ufs:" + " inodePath={}, options={}.", childURI, loadMetadataContext, e);
                    } catch (BlockInfoException | FileAlreadyCompletedException | FileDoesNotExistException | InvalidFileSizeException | IOException e) {
                        LOG.debug("Failed to loadMetadata because the ufs file or directory" + " is {}, options={}.", childStatus, loadMetadataContext, e);
                        failedSync++;
                    }
                }
                mInodeTree.setDirectChildrenLoaded(mRpcContext, inodePath.getInode().asDirectory());
            }
        }
    } catch (IOException | InterruptedException e) {
        LOG.debug("Failed to loadMetadata: inodePath={}, context={}.", inodePath.getUri(), context, e);
        throw new IOException(e);
    }
    if (failedSync > 0) {
        throw new IOException(String.format("Failed to load metadata of %s files or directories " + "under %s", failedSync, path));
    }
}

Example 79 with LockedInodePath

use of alluxio.master.file.meta.LockedInodePath in project alluxio by Alluxio.

the class DefaultFileSystemMaster method delete.

@Override
public void delete(AlluxioURI path, DeleteContext context) throws IOException, FileDoesNotExistException, DirectoryNotEmptyException, InvalidPathException, AccessControlException {
    if (isOperationComplete(context)) {
        Metrics.COMPLETED_OPERATION_RETRIED_COUNT.inc();
        LOG.warn("A completed \"delete\" operation has been retried. {}", context);
        return;
    }
    Metrics.DELETE_PATHS_OPS.inc();
    try (RpcContext rpcContext = createRpcContext(context);
        FileSystemMasterAuditContext auditContext = createAuditContext("delete", path, null, null)) {
        if (context.getOptions().getAlluxioOnly()) {
            LOG.debug("alluxio-only deletion on path {} skips metadata sync", path);
        } else {
            syncMetadata(rpcContext, path, context.getOptions().getCommonOptions(), context.getOptions().getRecursive() ? DescendantType.ALL : DescendantType.ONE, auditContext, LockedInodePath::getInodeOrNull, (inodePath, permChecker) -> permChecker.checkParentPermission(Mode.Bits.WRITE, inodePath), false);
        }
        LockingScheme lockingScheme = createLockingScheme(path, context.getOptions().getCommonOptions(), LockPattern.WRITE_EDGE);
        try (LockedInodePath inodePath = mInodeTree.lockInodePath(lockingScheme)) {
            mPermissionChecker.checkParentPermission(Mode.Bits.WRITE, inodePath);
            if (context.getOptions().getRecursive()) {
                List<String> failedChildren = new ArrayList<>();
                try (LockedInodePathList descendants = mInodeTree.getDescendants(inodePath)) {
                    for (LockedInodePath childPath : descendants) {
                        try {
                            mPermissionChecker.checkPermission(Mode.Bits.WRITE, childPath);
                            if (mMountTable.isMountPoint(childPath.getUri())) {
                                mMountTable.checkUnderWritableMountPoint(childPath.getUri());
                            }
                        } catch (AccessControlException e) {
                            failedChildren.add(e.getMessage());
                        }
                    }
                    if (failedChildren.size() > 0) {
                        throw new AccessControlException(ExceptionMessage.DELETE_FAILED_DIR_CHILDREN.getMessage(path, StringUtils.join(failedChildren, ",")));
                    }
                } catch (AccessControlException e) {
                    auditContext.setAllowed(false);
                    throw e;
                }
            }
            mMountTable.checkUnderWritableMountPoint(path);
            if (!inodePath.fullPathExists()) {
                throw new FileDoesNotExistException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(path));
            }
            deleteInternal(rpcContext, inodePath, context);
            auditContext.setSucceeded(true);
            cacheOperation(context);
        }
    }
}