Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 51 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method checkUfsMode.

/**
 * Check if the specified operation type is allowed to the ufs.
 *
 * @param alluxioPath the Alluxio path
 * @param opType the operation type
 */
private void checkUfsMode(AlluxioURI alluxioPath, OperationType opType) throws AccessControlException, InvalidPathException {
    MountTable.Resolution resolution = mMountTable.resolve(alluxioPath);
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        UfsMode ufsMode = ufs.getOperationMode(mUfsManager.getPhysicalUfsState(ufs.getPhysicalStores()));
        switch(ufsMode) {
            case NO_ACCESS:
                throw new AccessControlException(ExceptionMessage.UFS_OP_NOT_ALLOWED.getMessage(opType, resolution.getUri(), UfsMode.NO_ACCESS));
            case READ_ONLY:
                if (opType == OperationType.WRITE) {
                    throw new AccessControlException(ExceptionMessage.UFS_OP_NOT_ALLOWED.getMessage(opType, resolution.getUri(), UfsMode.READ_ONLY));
                }
                break;
            default:
                // All operations are allowed
                break;
        }
    }
}

Example 52 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getRootMountInfo.

private static MountInfo getRootMountInfo(MasterUfsManager ufsManager) {
    try (CloseableResource<UnderFileSystem> resource = ufsManager.getRoot().acquireUfsResource()) {
        boolean shared = resource.get().isObjectStorage() && ServerConfiguration.getBoolean(PropertyKey.UNDERFS_OBJECT_STORE_MOUNT_SHARED_PUBLICLY);
        boolean readonly = ServerConfiguration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_READONLY);
        String rootUfsUri = PathUtils.normalizePath(ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS), AlluxioURI.SEPARATOR);
        Map<String, String> rootUfsConf = ServerConfiguration.getNestedProperties(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION).entrySet().stream().filter(entry -> entry.getValue() != null).collect(Collectors.toMap(Map.Entry::getKey, entry -> String.valueOf(entry.getValue())));
        MountPOptions mountOptions = MountContext.mergeFrom(MountPOptions.newBuilder().setShared(shared).setReadOnly(readonly).putAllProperties(rootUfsConf)).getOptions().build();
        return new MountInfo(new AlluxioURI(MountTable.ROOT), new AlluxioURI(rootUfsUri), IdUtils.ROOT_MOUNT_ID, mountOptions);
    }
}

Example 53 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method completeFileInternal.

/**
 * Completes a file. After a file is completed, it cannot be written to.
 *
 * @param rpcContext the rpc context
 * @param inodePath the {@link LockedInodePath} to complete
 * @param context the method context
 */
void completeFileInternal(RpcContext rpcContext, LockedInodePath inodePath, CompleteFileContext context) throws InvalidPathException, FileDoesNotExistException, BlockInfoException, FileAlreadyCompletedException, InvalidFileSizeException, UnavailableException {
    Inode inode = inodePath.getInode();
    if (!inode.isFile()) {
        throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(inodePath.getUri()));
    }
    InodeFile fileInode = inode.asFile();
    List<Long> blockIdList = fileInode.getBlockIds();
    List<BlockInfo> blockInfoList = mBlockMaster.getBlockInfoList(blockIdList);
    if (!fileInode.isPersisted() && blockInfoList.size() != blockIdList.size()) {
        throw new BlockInfoException("Cannot complete a file without all the blocks committed");
    }
    // Iterate over all file blocks committed to Alluxio, computing the length and verify that all
    // the blocks (except the last one) is the same size as the file block size.
    long inAlluxioLength = 0;
    long fileBlockSize = fileInode.getBlockSizeBytes();
    for (int i = 0; i < blockInfoList.size(); i++) {
        BlockInfo blockInfo = blockInfoList.get(i);
        inAlluxioLength += blockInfo.getLength();
        if (i < blockInfoList.size() - 1 && blockInfo.getLength() != fileBlockSize) {
            throw new BlockInfoException("Block index " + i + " has a block size smaller than the file block size (" + fileInode.getBlockSizeBytes() + ")");
        }
    }
    // If the file is persisted, its length is determined by UFS. Otherwise, its length is
    // determined by its size in Alluxio.
    long length = fileInode.isPersisted() ? context.getOptions().getUfsLength() : inAlluxioLength;
    String ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
    if (fileInode.isPersisted()) {
        UfsStatus ufsStatus = context.getUfsStatus();
        // Retrieve the UFS fingerprint for this file.
        MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
        AlluxioURI resolvedUri = resolution.getUri();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            if (ufsStatus == null) {
                ufsFingerprint = ufs.getFingerprint(resolvedUri.toString());
            } else {
                ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
            }
        }
    }
    completeFileInternal(rpcContext, inodePath, length, context.getOperationTimeMs(), ufsFingerprint);
}

Example 54 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class InodeSyncStream method loadFileMetadataInternal.

/**
 * Loads metadata for the file identified by the given path from UFS into Alluxio.
 *
 * This method doesn't require any specific type of locking on inodePath. If the path needs to be
 * loaded, we will acquire a write-edge lock.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path for which metadata should be loaded
 * @param resolution the UFS resolution of path
 * @param context the load metadata context
 */
static void loadFileMetadataInternal(RpcContext rpcContext, LockedInodePath inodePath, MountTable.Resolution resolution, LoadMetadataContext context, DefaultFileSystemMaster fsMaster) throws BlockInfoException, FileDoesNotExistException, InvalidPathException, FileAlreadyCompletedException, InvalidFileSizeException, IOException {
    if (inodePath.fullPathExists()) {
        return;
    }
    AlluxioURI ufsUri = resolution.getUri();
    long ufsBlockSizeByte;
    long ufsLength;
    AccessControlList acl = null;
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        if (context.getUfsStatus() == null) {
            context.setUfsStatus(ufs.getExistingFileStatus(ufsUri.toString()));
        }
        ufsLength = ((UfsFileStatus) context.getUfsStatus()).getContentLength();
        long blockSize = ((UfsFileStatus) context.getUfsStatus()).getBlockSize();
        ufsBlockSizeByte = blockSize != UfsFileStatus.UNKNOWN_BLOCK_SIZE ? blockSize : ufs.getBlockSizeByte(ufsUri.toString());
        if (fsMaster.isAclEnabled()) {
            Pair<AccessControlList, DefaultAccessControlList> aclPair = ufs.getAclPair(ufsUri.toString());
            if (aclPair != null) {
                acl = aclPair.getFirst();
                // DefaultACL should be null, because it is a file
                if (aclPair.getSecond() != null) {
                    LOG.warn("File {} has default ACL in the UFS", inodePath.getUri());
                }
            }
        }
    }
    // Metadata loaded from UFS has no TTL set.
    CreateFileContext createFileContext = CreateFileContext.defaults();
    createFileContext.getOptions().setBlockSizeBytes(ufsBlockSizeByte);
    createFileContext.getOptions().setRecursive(context.getOptions().getCreateAncestors());
    createFileContext.getOptions().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(context.getOptions().getCommonOptions().getTtl()).setTtlAction(context.getOptions().getCommonOptions().getTtlAction()));
    // set as through since already in UFS
    createFileContext.setWriteType(WriteType.THROUGH);
    createFileContext.setMetadataLoad(true);
    createFileContext.setOwner(context.getUfsStatus().getOwner());
    createFileContext.setGroup(context.getUfsStatus().getGroup());
    createFileContext.setXAttr(context.getUfsStatus().getXAttr());
    short ufsMode = context.getUfsStatus().getMode();
    Mode mode = new Mode(ufsMode);
    Long ufsLastModified = context.getUfsStatus().getLastModifiedTime();
    if (resolution.getShared()) {
        mode.setOtherBits(mode.getOtherBits().or(mode.getOwnerBits()));
    }
    createFileContext.getOptions().setMode(mode.toProto());
    if (acl != null) {
        createFileContext.setAcl(acl.getEntries());
    }
    if (ufsLastModified != null) {
        createFileContext.setOperationTimeMs(ufsLastModified);
    }
    try (LockedInodePath writeLockedPath = inodePath.lockFinalEdgeWrite();
        MergeJournalContext merger = new MergeJournalContext(rpcContext.getJournalContext(), writeLockedPath.getUri(), InodeSyncStream::mergeCreateComplete)) {
        // We do not want to close this wrapRpcContext because it uses elements from another context
        RpcContext wrapRpcContext = new RpcContext(rpcContext.getBlockDeletionContext(), merger, rpcContext.getOperationContext());
        fsMaster.createFileInternal(wrapRpcContext, writeLockedPath, createFileContext);
        CompleteFileContext completeContext = CompleteFileContext.mergeFrom(CompleteFilePOptions.newBuilder().setUfsLength(ufsLength)).setUfsStatus(context.getUfsStatus());
        if (ufsLastModified != null) {
            completeContext.setOperationTimeMs(ufsLastModified);
        }
        fsMaster.completeFileInternal(wrapRpcContext, writeLockedPath, completeContext);
    } catch (FileAlreadyExistsException e) {
        // This may occur if a thread created or loaded the file before we got the write lock.
        // The file already exists, so nothing needs to be loaded.
        LOG.debug("Failed to load file metadata: {}", e.toString());
    }
    // Re-traverse the path to pick up any newly created inodes.
    inodePath.traverse();
}

Example 55 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class ActiveSyncManager method start.

/**
 * Start the polling threads.
 */
public void start() throws IOException {
    mStarted = true;
    // Initialize UFS states
    for (AlluxioURI syncPoint : mSyncPathList) {
        MountTable.Resolution resolution;
        try {
            resolution = mMountTable.resolve(syncPoint);
        } catch (InvalidPathException e) {
            LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " + "path {}, exception {}", syncPoint, e);
            continue;
        }
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            if (!ufsResource.get().supportsActiveSync()) {
                throw new UnsupportedOperationException("Active Sync is not supported on this UFS type: " + ufsResource.get().getUnderFSType());
            }
            ufsResource.get().startSync(resolution.getUri());
        }
    }
    // therefore forces a sync
    for (Map.Entry<Long, List<AlluxioURI>> entry : mFilterMap.entrySet()) {
        long mountId = entry.getKey();
        long txId = mStartingTxIdMap.getOrDefault(mountId, SyncInfo.INVALID_TXID);
        if (!entry.getValue().isEmpty()) {
            launchPollingThread(mountId, txId);
        }
        try {
            if ((txId == SyncInfo.INVALID_TXID) && ServerConfiguration.getBoolean(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INITIAL_SYNC_ENABLED)) {
                mExecutorService.submit(() -> entry.getValue().parallelStream().forEach(syncPoint -> {
                    MountTable.Resolution resolution;
                    try {
                        resolution = mMountTable.resolve(syncPoint);
                    } catch (InvalidPathException e) {
                        LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " + "path {}, exception {}", syncPoint, e);
                        return;
                    }
                    startInitialFullSync(syncPoint, resolution);
                }));
            }
        } catch (Exception e) {
            LOG.warn("exception encountered during initial sync: {}", e.toString());
        }
    }
}