Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 36 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class InodeSyncStream method loadDirectoryMetadata.

/**
 * Loads metadata for the directory identified by the given path from UFS into Alluxio. This does
 * not actually require looking at the UFS path.
 * It is a no-op if the directory exists.
 *
 * This method doesn't require any specific type of locking on inodePath. If the path needs to be
 * loaded, we will acquire a write-edge lock if necessary.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path for which metadata should be loaded
 * @param context the load metadata context
 */
static void loadDirectoryMetadata(RpcContext rpcContext, LockedInodePath inodePath, LoadMetadataContext context, MountTable mountTable, DefaultFileSystemMaster fsMaster) throws FileDoesNotExistException, InvalidPathException, AccessControlException, IOException {
    if (inodePath.fullPathExists()) {
        return;
    }
    CreateDirectoryContext createDirectoryContext = CreateDirectoryContext.defaults();
    createDirectoryContext.getOptions().setRecursive(context.getOptions().getCreateAncestors()).setAllowExists(false).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction()));
    createDirectoryContext.setMountPoint(mountTable.isMountPoint(inodePath.getUri()));
    createDirectoryContext.setMetadataLoad(true);
    createDirectoryContext.setWriteType(WriteType.THROUGH);
    MountTable.Resolution resolution = mountTable.resolve(inodePath.getUri());
    AlluxioURI ufsUri = resolution.getUri();
    AccessControlList acl = null;
    DefaultAccessControlList defaultAcl = null;
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        if (context.getUfsStatus() == null) {
            context.setUfsStatus(ufs.getExistingDirectoryStatus(ufsUri.toString()));
        }
        Pair<AccessControlList, DefaultAccessControlList> aclPair = ufs.getAclPair(ufsUri.toString());
        if (aclPair != null) {
            acl = aclPair.getFirst();
            defaultAcl = aclPair.getSecond();
        }
    }
    String ufsOwner = context.getUfsStatus().getOwner();
    String ufsGroup = context.getUfsStatus().getGroup();
    short ufsMode = context.getUfsStatus().getMode();
    Long lastModifiedTime = context.getUfsStatus().getLastModifiedTime();
    Mode mode = new Mode(ufsMode);
    if (resolution.getShared()) {
        mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits()));
    }
    createDirectoryContext.getOptions().setMode(mode.toProto());
    createDirectoryContext.setOwner(ufsOwner).setGroup(ufsGroup).setUfsStatus(context.getUfsStatus());
    createDirectoryContext.setXAttr(context.getUfsStatus().getXAttr());
    if (acl != null) {
        createDirectoryContext.setAcl(acl.getEntries());
    }
    if (defaultAcl != null) {
        createDirectoryContext.setDefaultAcl(defaultAcl.getEntries());
    }
    if (lastModifiedTime != null) {
        createDirectoryContext.setOperationTimeMs(lastModifiedTime);
    }
    try (LockedInodePath writeLockedPath = inodePath.lockFinalEdgeWrite()) {
        fsMaster.createDirectoryInternal(rpcContext, writeLockedPath, createDirectoryContext);
    } catch (FileAlreadyExistsException e) {
    // This may occur if a thread created or loaded the directory before we got the write lock.
    // The directory already exists, so nothing needs to be loaded.
    }
    // Re-traverse the path to pick up any newly created inodes.
    inodePath.traverse();
}

Example 37 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class InodeSyncStream method syncExistingInodeMetadata.

/**
 * Sync inode metadata with the UFS state.
 *
 * This method expects the {@code inodePath} to already exist in the inode tree.
 */
private void syncExistingInodeMetadata(LockedInodePath inodePath, boolean skipLoad) throws AccessControlException, BlockInfoException, FileAlreadyCompletedException, FileDoesNotExistException, InvalidFileSizeException, InvalidPathException, IOException, InterruptedException {
    if (inodePath.getLockPattern() != LockPattern.WRITE_EDGE && !mLoadOnly) {
        throw new RuntimeException(String.format("syncExistingInodeMetadata was called on d%s when only locked with %s. Load metadata" + " only was not specified.", inodePath.getUri(), inodePath.getLockPattern()));
    }
    // Set to true if the given inode was deleted.
    boolean deletedInode = false;
    // whether we need to load metadata for the current path
    boolean loadMetadata = mLoadOnly;
    LOG.trace("Syncing inode metadata {}", inodePath.getUri());
    // The requested path already exists in Alluxio.
    Inode inode = inodePath.getInode();
    // initialize sync children to true if it is a listStatus call on a directory
    boolean syncChildren = inode.isDirectory() && !mIsGetFileInfo && !mDescendantType.equals(DescendantType.NONE);
    if (inodePath.getLockPattern() == LockPattern.WRITE_EDGE && !mLoadOnly) {
        if (inode instanceof InodeFile && !inode.asFile().isCompleted()) {
            // Do not sync an incomplete file, since the UFS file is expected to not exist.
            return;
        }
        Optional<Scoped> persistingLock = mInodeLockManager.tryAcquirePersistingLock(inode.getId());
        if (!persistingLock.isPresent()) {
            // written.
            return;
        }
        persistingLock.get().close();
        UfsStatus cachedStatus = null;
        boolean fileNotFound = false;
        try {
            cachedStatus = mStatusCache.getStatus(inodePath.getUri());
        } catch (FileNotFoundException e) {
            fileNotFound = true;
        }
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        AlluxioURI ufsUri = resolution.getUri();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            String ufsFingerprint;
            Fingerprint ufsFpParsed;
            // When the status is not cached and it was not due to file not found, we retry
            if (fileNotFound) {
                ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
                ufsFpParsed = Fingerprint.parse(ufsFingerprint);
            } else if (cachedStatus == null) {
                // TODO(david): change the interface so that getFingerprint returns a parsed fingerprint
                ufsFingerprint = (String) getFromUfs(() -> ufs.getFingerprint(ufsUri.toString()));
                ufsFpParsed = Fingerprint.parse(ufsFingerprint);
            } else {
                // When the status is cached
                Pair<AccessControlList, DefaultAccessControlList> aclPair = (Pair<AccessControlList, DefaultAccessControlList>) getFromUfs(() -> ufs.getAclPair(ufsUri.toString()));
                if (aclPair == null || aclPair.getFirst() == null || !aclPair.getFirst().hasExtended()) {
                    ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus);
                } else {
                    ufsFpParsed = Fingerprint.create(ufs.getUnderFSType(), cachedStatus, aclPair.getFirst());
                }
                ufsFingerprint = ufsFpParsed.serialize();
            }
            boolean containsMountPoint = mMountTable.containsMountPoint(inodePath.getUri(), true);
            UfsSyncUtils.SyncPlan syncPlan = UfsSyncUtils.computeSyncPlan(inode, ufsFpParsed, containsMountPoint);
            if (syncPlan.toUpdateMetaData()) {
                // It works by calling SetAttributeInternal on the inodePath.
                if (ufsFpParsed != null && ufsFpParsed.isValid()) {
                    short mode = Short.parseShort(ufsFpParsed.getTag(Fingerprint.Tag.MODE));
                    long opTimeMs = System.currentTimeMillis();
                    SetAttributePOptions.Builder builder = SetAttributePOptions.newBuilder().setMode(new Mode(mode).toProto());
                    String owner = ufsFpParsed.getTag(Fingerprint.Tag.OWNER);
                    if (!owner.equals(Fingerprint.UNDERSCORE)) {
                        // Only set owner if not empty
                        builder.setOwner(owner);
                    }
                    String group = ufsFpParsed.getTag(Fingerprint.Tag.GROUP);
                    if (!group.equals(Fingerprint.UNDERSCORE)) {
                        // Only set group if not empty
                        builder.setGroup(group);
                    }
                    SetAttributeContext ctx = SetAttributeContext.mergeFrom(builder).setUfsFingerprint(ufsFingerprint);
                    mFsMaster.setAttributeSingleFile(mRpcContext, inodePath, false, opTimeMs, ctx);
                }
            }
            if (syncPlan.toDelete()) {
                deletedInode = true;
                try {
                    // The options for deleting.
                    DeleteContext syncDeleteContext = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true).setAlluxioOnly(true).setUnchecked(true));
                    mFsMaster.deleteInternal(mRpcContext, inodePath, syncDeleteContext);
                } catch (DirectoryNotEmptyException | IOException e) {
                    // Should not happen, since it is an unchecked delete.
                    LOG.error("Unexpected error for unchecked delete.", e);
                }
            }
            if (syncPlan.toLoadMetadata()) {
                loadMetadata = true;
            }
            syncChildren = syncPlan.toSyncChildren();
        }
    }
    // if DescendantType.ONE we only sync children if we are syncing root of this stream
    if (mDescendantType == DescendantType.ONE) {
        syncChildren = syncChildren && mRootScheme.getPath().equals(inodePath.getUri());
    }
    Map<String, Inode> inodeChildren = new HashMap<>();
    if (syncChildren) {
        // maps children name to inode
        mInodeStore.getChildren(inode.asDirectory()).forEach(child -> inodeChildren.put(child.getName(), child));
        // Fetch and populate children into the cache
        mStatusCache.prefetchChildren(inodePath.getUri(), mMountTable);
        Collection<UfsStatus> listStatus = mStatusCache.fetchChildrenIfAbsent(mRpcContext, inodePath.getUri(), mMountTable);
        // Iterate over UFS listings and process UFS children.
        if (listStatus != null) {
            for (UfsStatus ufsChildStatus : listStatus) {
                if (!inodeChildren.containsKey(ufsChildStatus.getName()) && !PathUtils.isTemporaryFileName(ufsChildStatus.getName())) {
                    // Ufs child exists, but Alluxio child does not. Must load metadata.
                    loadMetadata = true;
                    break;
                }
            }
        }
    }
    // locked path
    if (deletedInode) {
        inodePath.removeLastInode();
    }
    // load metadata if necessary.
    if (loadMetadata && !skipLoad) {
        loadMetadataForPath(inodePath);
    }
    if (syncChildren) {
        // Iterate over Alluxio children and process persisted children.
        mInodeStore.getChildren(inode.asDirectory()).forEach(childInode -> {
            // its children.
            if (mLoadOnly && inodeChildren.containsKey(childInode.getName()) && childInode.isFile()) {
                return;
            }
            // If we're performing a recursive sync, add each child of our current Inode to the queue
            AlluxioURI child = inodePath.getUri().joinUnsafe(childInode.getName());
            mPendingPaths.add(child);
            // This asynchronously schedules a job to pre-fetch the statuses into the cache.
            if (childInode.isDirectory() && mDescendantType == DescendantType.ALL) {
                mStatusCache.prefetchChildren(child, mMountTable);
            }
        });
    }
}

Example 38 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class InodeSyncStream method loadFileMetadataInternal.

/**
 * Loads metadata for the file identified by the given path from UFS into Alluxio.
 *
 * This method doesn't require any specific type of locking on inodePath. If the path needs to be
 * loaded, we will acquire a write-edge lock.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path for which metadata should be loaded
 * @param resolution the UFS resolution of path
 * @param context the load metadata context
 */
static void loadFileMetadataInternal(RpcContext rpcContext, LockedInodePath inodePath, MountTable.Resolution resolution, LoadMetadataContext context, DefaultFileSystemMaster fsMaster) throws BlockInfoException, FileDoesNotExistException, InvalidPathException, FileAlreadyCompletedException, InvalidFileSizeException, IOException {
    if (inodePath.fullPathExists()) {
        return;
    }
    AlluxioURI ufsUri = resolution.getUri();
    long ufsBlockSizeByte;
    long ufsLength;
    AccessControlList acl = null;
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        if (context.getUfsStatus() == null) {
            context.setUfsStatus(ufs.getExistingFileStatus(ufsUri.toString()));
        }
        ufsLength = ((UfsFileStatus) context.getUfsStatus()).getContentLength();
        long blockSize = ((UfsFileStatus) context.getUfsStatus()).getBlockSize();
        ufsBlockSizeByte = blockSize != UfsFileStatus.UNKNOWN_BLOCK_SIZE ? blockSize : ufs.getBlockSizeByte(ufsUri.toString());
        if (fsMaster.isAclEnabled()) {
            Pair<AccessControlList, DefaultAccessControlList> aclPair = ufs.getAclPair(ufsUri.toString());
            if (aclPair != null) {
                acl = aclPair.getFirst();
                // DefaultACL should be null, because it is a file
                if (aclPair.getSecond() != null) {
                    LOG.warn("File {} has default ACL in the UFS", inodePath.getUri());
                }
            }
        }
    }
    // Metadata loaded from UFS has no TTL set.
    CreateFileContext createFileContext = CreateFileContext.defaults();
    createFileContext.getOptions().setBlockSizeBytes(ufsBlockSizeByte);
    createFileContext.getOptions().setRecursive(context.getOptions().getCreateAncestors());
    createFileContext.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction()));
    // set as through since already in UFS
    createFileContext.setWriteType(WriteType.THROUGH);
    createFileContext.setMetadataLoad(true);
    createFileContext.setOwner(context.getUfsStatus().getOwner());
    createFileContext.setGroup(context.getUfsStatus().getGroup());
    createFileContext.setXAttr(context.getUfsStatus().getXAttr());
    short ufsMode = context.getUfsStatus().getMode();
    Mode mode = new Mode(ufsMode);
    Long ufsLastModified = context.getUfsStatus().getLastModifiedTime();
    if (resolution.getShared()) {
        mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits()));
    }
    createFileContext.getOptions().setMode(mode.toProto());
    if (acl != null) {
        createFileContext.setAcl(acl.getEntries());
    }
    if (ufsLastModified != null) {
        createFileContext.setOperationTimeMs(ufsLastModified);
    }
    try (LockedInodePath writeLockedPath = inodePath.lockFinalEdgeWrite();
        MergeJournalContext merger = new MergeJournalContext(rpcContext.getJournalContext(), writeLockedPath.getUri(), InodeSyncStream::mergeCreateComplete)) {
        // We do not want to close this wrapRpcContext because it uses elements from another context
        RpcContext wrapRpcContext = new RpcContext(rpcContext.getBlockDeletionContext(), merger, rpcContext.getOperationContext());
        fsMaster.createFileInternal(wrapRpcContext, writeLockedPath, createFileContext);
        CompleteFileContext completeContext = CompleteFileContext.mergeFrom(CompleteFilePOptions.newBuilder().setUfsLength(ufsLength)).setUfsStatus(context.getUfsStatus());
        if (ufsLastModified != null) {
            completeContext.setOperationTimeMs(ufsLastModified);
        }
        fsMaster.completeFileInternal(wrapRpcContext, writeLockedPath, completeContext);
    } catch (FileAlreadyExistsException e) {
        // This may occur if a thread created or loaded the file before we got the write lock.
        // The file already exists, so nothing needs to be loaded.
        LOG.debug("Failed to load file metadata: {}", e.toString());
    }
    // Re-traverse the path to pick up any newly created inodes.
    inodePath.traverse();
}

Example 39 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class AsyncUfsAbsentPathCache method processSinglePath.

/**
 * Processes and checks the existence of the corresponding ufs path for the given Alluxio path.
 *
 * @param alluxioUri the Alluxio path to process
 * @param mountInfo the associated {@link MountInfo} for the Alluxio path
 * @return if true, further traversal of the descendant paths should continue
 */
private boolean processSinglePath(AlluxioURI alluxioUri, MountInfo mountInfo) {
    PathLock pathLock = new PathLock();
    Lock writeLock = pathLock.writeLock();
    Lock readLock = null;
    try {
        // Write lock this path, to only enable a single task per path
        writeLock.lock();
        PathLock existingLock = mCurrentPaths.putIfAbsent(alluxioUri.getPath(), pathLock);
        if (existingLock != null) {
            // Another thread already locked this path and is processing it. Wait for the other
            // thread to finish, by locking the existing read lock.
            writeLock.unlock();
            writeLock = null;
            readLock = existingLock.readLock();
            readLock.lock();
            if (mCache.getIfPresent(alluxioUri.getPath()) != null) {
                // This path is already in the cache (is absent). Further traversal is unnecessary.
                return false;
            }
        } else {
            // This thread has the exclusive lock for this path.
            // Resolve this Alluxio uri. It should match the original mount id.
            MountTable.Resolution resolution = mMountTable.resolve(alluxioUri);
            if (resolution.getMountId() != mountInfo.getMountId()) {
                // This mount point has changed. Further traversal is unnecessary.
                return false;
            }
            boolean existsInUfs;
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                existsInUfs = ufs.exists(resolution.getUri().toString());
            }
            if (existsInUfs) {
                // This ufs path exists. Remove the cache entry.
                removeCacheEntry(alluxioUri.getPath());
            } else {
                // This is the first ufs path which does not exist. Add it to the cache.
                addCacheEntry(alluxioUri.getPath(), mountInfo);
                if (pathLock.isInvalidate()) {
                    // This path was marked to be invalidated, meaning this UFS path was just created,
                    // and now exists. Invalidate the entry.
                    // This check is necessary to avoid the race with the invalidating thread.
                    removeCacheEntry(alluxioUri.getPath());
                } else {
                    // Further traversal is unnecessary.
                    return false;
                }
            }
        }
    } catch (InvalidPathException | IOException e) {
        LOG.warn("Processing path failed: " + alluxioUri, e);
        return false;
    } finally {
        // Unlock the path
        if (readLock != null) {
            readLock.unlock();
        }
        if (writeLock != null) {
            mCurrentPaths.remove(alluxioUri.getPath(), pathLock);
            writeLock.unlock();
        }
    }
    return true;
}

Example 40 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class ActiveSyncer method heartbeat.

@Override
public void heartbeat() {
    LOG.debug("start sync heartbeat for {} with mount id {}", mMountUri, mMountId);
    // Remove any previously completed sync tasks
    mSyncTasks.removeIf(Future::isDone);
    List<AlluxioURI> filterList = mSyncManager.getFilterList(mMountId);
    if (filterList == null || filterList.isEmpty()) {
        return;
    }
    try {
        UfsManager.UfsClient ufsclient = Objects.requireNonNull(mMountTable.getUfsClient(mMountId));
        try (CloseableResource<UnderFileSystem> ufsResource = ufsclient.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            if (!ufs.supportsActiveSync()) {
                return;
            }
            SyncInfo syncInfo = ufs.getActiveSyncInfo();
            // This returns a list of ufsUris that we need to sync.
            Set<AlluxioURI> ufsSyncPoints = syncInfo.getSyncPoints();
            // Parallelize across sync points
            List<CompletableFuture<Long>> tasksPerSync = new ArrayList<>();
            for (AlluxioURI ufsUri : ufsSyncPoints) {
                tasksPerSync.add(CompletableFuture.supplyAsync(() -> {
                    processSyncPoint(ufsUri, syncInfo);
                    return syncInfo.getTxId();
                }, mSyncManager.getExecutor()));
            }
            // Journal the latest processed txId
            CompletableFuture<Void> syncTask = CompletableFuture.allOf(tasksPerSync.toArray(new CompletableFuture<?>[0])).thenRunAsync(() -> mFileSystemMaster.recordActiveSyncTxid(syncInfo.getTxId(), mMountId), mSyncManager.getExecutor());
            int attempts = 0;
            while (!mSyncTasks.offer(syncTask)) {
                if (Thread.currentThread().isInterrupted()) {
                    break;
                }
                // sync tasks in the last 32 / (heartbeats/minute) minutes have not completed.
                for (CompletableFuture<?> f : mSyncTasks) {
                    try {
                        long waitTime = ServerConfiguration.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INTERVAL) / mSyncTasks.size();
                        f.get(waitTime, TimeUnit.MILLISECONDS);
                        mSyncTasks.remove(f);
                        break;
                    } catch (TimeoutException e) {
                        LOG.trace("sync task did not complete during heartbeat. Attempt: {}", attempts);
                    } catch (InterruptedException | ExecutionException e) {
                        LogUtils.warnWithException(LOG, "Failed while waiting on task to add new task to " + "head of queue", e);
                        if (Thread.currentThread().isInterrupted()) {
                            Thread.currentThread().interrupt();
                        }
                    }
                    attempts++;
                }
            }
        }
    } catch (IOException e) {
        LOG.warn("IOException " + Throwables.getStackTraceAsString(e));
    }
}