Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 56 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class ActiveSyncManager method launchPollingThread.

/**
 * Launches polling thread on a particular mount point with starting txId.
 *
 * @param mountId launch polling thread on a mount id
 * @param txId specifies the transaction id to initialize the pollling thread
 */
public void launchPollingThread(long mountId, long txId) {
    LOG.debug("launch polling thread for mount id {}, txId {}", mountId, txId);
    if (!mPollerMap.containsKey(mountId)) {
        UfsManager.UfsClient ufsClient = mMountTable.getUfsClient(mountId);
        if (ufsClient == null) {
            LOG.warn("Mount id {} does not exist", mountId);
            return;
        }
        try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) {
            ufsResource.get().startActiveSyncPolling(txId);
        } catch (IOException e) {
            LOG.warn("IO Exception trying to launch Polling thread: {}", e.toString());
        }
        ActiveSyncer syncer = new ActiveSyncer(mFileSystemMaster, this, mMountTable, mountId);
        Future<?> future = getExecutor().submit(new HeartbeatThread(HeartbeatContext.MASTER_ACTIVE_UFS_SYNC, syncer, (int) ServerConfiguration.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INTERVAL), ServerConfiguration.global(), ServerUserState.global()));
        mPollerMap.put(mountId, future);
    }
}

Example 57 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class ActiveSyncManager method startSyncAndJournal.

/**
 * Start active sync on a URI and journal the add entry.
 *
 * @param rpcContext the master rpc or no-op context
 * @param syncPoint sync point to be start
 */
public void startSyncAndJournal(RpcContext rpcContext, AlluxioURI syncPoint) throws InvalidPathException {
    try (LockResource r = new LockResource(mLock)) {
        MountTable.Resolution resolution = mMountTable.resolve(syncPoint);
        long mountId = resolution.getMountId();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            if (!ufsResource.get().supportsActiveSync()) {
                throw new UnsupportedOperationException("Active Syncing is not supported on this UFS type: " + ufsResource.get().getUnderFSType());
            }
        }
        if (isUnderSyncPoint(syncPoint)) {
            throw new InvalidPathException("URI " + syncPoint + " is already a sync point");
        }
        AddSyncPointEntry addSyncPoint = AddSyncPointEntry.newBuilder().setSyncpointPath(syncPoint.toString()).setMountId(mountId).build();
        applyAndJournal(rpcContext, addSyncPoint);
        try {
            startSyncInternal(syncPoint, resolution);
        } catch (Throwable e) {
            LOG.warn("Start sync failed on {}", syncPoint, e);
            // revert state;
            RemoveSyncPointEntry removeSyncPoint = File.RemoveSyncPointEntry.newBuilder().setSyncpointPath(syncPoint.toString()).build();
            applyAndJournal(rpcContext, removeSyncPoint);
            recoverFromStartSync(syncPoint, resolution.getMountId());
            throw e;
        }
    }
}

Example 58 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method createDirectoryInternal.

/**
 * Implementation of directory creation for a given path.
 *
 * @param rpcContext the rpc context
 * @param inodePath the path of the directory
 * @param context method context
 * @return a list of created inodes
 */
List<Inode> createDirectoryInternal(RpcContext rpcContext, LockedInodePath inodePath, CreateDirectoryContext context) throws InvalidPathException, FileAlreadyExistsException, IOException, FileDoesNotExistException {
    Preconditions.checkState(inodePath.getLockPattern() == LockPattern.WRITE_EDGE);
    try {
        List<Inode> createResult = mInodeTree.createPath(rpcContext, inodePath, context);
        InodeDirectory inodeDirectory = inodePath.getInode().asDirectory();
        String ufsFingerprint = Constants.INVALID_UFS_FINGERPRINT;
        if (inodeDirectory.isPersisted()) {
            UfsStatus ufsStatus = context.getUfsStatus();
            // Retrieve the UFS fingerprint for this file.
            MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
            AlluxioURI resolvedUri = resolution.getUri();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                if (ufsStatus == null) {
                    ufsFingerprint = ufs.getFingerprint(resolvedUri.toString());
                } else {
                    ufsFingerprint = Fingerprint.create(ufs.getUnderFSType(), ufsStatus).serialize();
                }
            }
        }
        mInodeTree.updateInode(rpcContext, UpdateInodeEntry.newBuilder().setId(inodeDirectory.getId()).setUfsFingerprint(ufsFingerprint).build());
        if (context.isPersisted()) {
            // The path exists in UFS, so it is no longer absent.
            mUfsAbsentPathCache.processExisting(inodePath.getUri());
        }
        Metrics.DIRECTORIES_CREATED.inc();
        return createResult;
    } catch (BlockInfoException e) {
        // happen.
        throw new RuntimeException(e);
    }
}

Example 59 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method setUfsAcl.

private void setUfsAcl(LockedInodePath inodePath) throws InvalidPathException, AccessControlException {
    Inode inode = inodePath.getInodeOrNull();
    checkUfsMode(inodePath.getUri(), OperationType.WRITE);
    MountTable.Resolution resolution = mMountTable.resolve(inodePath.getUri());
    String ufsUri = resolution.getUri().toString();
    try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        if (ufs.isObjectStorage()) {
            LOG.warn("SetACL is not supported to object storage UFS via Alluxio. " + "UFS: " + ufsUri + ". This has no effect on the underlying object.");
        } else {
            try {
                List<AclEntry> entries = new ArrayList<>(inode.getACL().getEntries());
                if (inode.isDirectory()) {
                    entries.addAll(inode.asDirectory().getDefaultACL().getEntries());
                }
                ufs.setAclEntries(ufsUri, entries);
            } catch (IOException e) {
                throw new AccessControlException("Could not setAcl for UFS file: " + ufsUri);
            }
        }
    }
}

Example 60 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method renameInternal.

/**
 * Implements renaming.
 *
 * @param rpcContext the rpc context
 * @param srcInodePath the path of the rename source
 * @param dstInodePath the path to the rename destination
 * @param replayed whether the operation is a result of replaying the journal
 * @param context method options
 */
private void renameInternal(RpcContext rpcContext, LockedInodePath srcInodePath, LockedInodePath dstInodePath, boolean replayed, RenameContext context) throws FileDoesNotExistException, InvalidPathException, IOException, AccessControlException {
    // Rename logic:
    // 1. Change the source inode name to the destination name.
    // 2. Insert the source inode into the destination parent.
    // 3. Do UFS operations if necessary.
    // 4. Remove the source inode (reverting the name) from the source parent.
    // 5. Set the last modification times for both source and destination parent inodes.
    Inode srcInode = srcInodePath.getInode();
    AlluxioURI srcPath = srcInodePath.getUri();
    AlluxioURI dstPath = dstInodePath.getUri();
    InodeDirectory srcParentInode = srcInodePath.getParentInodeDirectory();
    InodeDirectory dstParentInode = dstInodePath.getParentInodeDirectory();
    String srcName = srcPath.getName();
    String dstName = dstPath.getName();
    LOG.debug("Renaming {} to {}", srcPath, dstPath);
    if (dstInodePath.fullPathExists()) {
        throw new InvalidPathException("Destination path: " + dstPath + " already exists.");
    }
    mInodeTree.rename(rpcContext, RenameEntry.newBuilder().setId(srcInode.getId()).setOpTimeMs(context.getOperationTimeMs()).setNewParentId(dstParentInode.getId()).setNewName(dstName).setPath(srcPath.getPath()).setNewPath(dstPath.getPath()).build());
    // If the source file is persisted, rename it in the UFS.
    try {
        if (!replayed && srcInode.isPersisted()) {
            // Check if ufs is writable
            checkUfsMode(srcPath, OperationType.WRITE);
            checkUfsMode(dstPath, OperationType.WRITE);
            MountTable.Resolution resolution = mMountTable.resolve(srcPath);
            // Persist ancestor directories from top to the bottom. We cannot use recursive create
            // parents here because the permission for the ancestors can be different.
            // inodes from the same mount point as the dst
            Stack<InodeDirectory> sameMountDirs = new Stack<>();
            List<Inode> dstInodeList = dstInodePath.getInodeList();
            for (int i = dstInodeList.size() - 1; i >= 0; i--) {
                // Since dstInodePath is guaranteed not to be a full path, all inodes in the incomplete
                // path are guaranteed to be a directory.
                InodeDirectory dir = dstInodeList.get(i).asDirectory();
                sameMountDirs.push(dir);
                if (dir.isMountPoint()) {
                    break;
                }
            }
            while (!sameMountDirs.empty()) {
                InodeDirectory dir = sameMountDirs.pop();
                if (!dir.isPersisted()) {
                    mInodeTree.syncPersistExistingDirectory(rpcContext, dir);
                }
            }
            String ufsSrcPath = resolution.getUri().toString();
            try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
                UnderFileSystem ufs = ufsResource.get();
                String ufsDstUri = mMountTable.resolve(dstPath).getUri().toString();
                boolean success;
                if (srcInode.isFile()) {
                    success = ufs.renameRenamableFile(ufsSrcPath, ufsDstUri);
                } else {
                    success = ufs.renameRenamableDirectory(ufsSrcPath, ufsDstUri);
                }
                if (!success) {
                    throw new IOException(ExceptionMessage.FAILED_UFS_RENAME.getMessage(ufsSrcPath, ufsDstUri));
                }
            }
            // The destination was persisted in ufs.
            mUfsAbsentPathCache.processExisting(dstPath);
        }
    } catch (Throwable t) {
        // On failure, revert changes and throw exception.
        mInodeTree.rename(rpcContext, RenameEntry.newBuilder().setId(srcInode.getId()).setOpTimeMs(context.getOperationTimeMs()).setNewName(srcName).setNewParentId(srcParentInode.getId()).setPath(dstPath.getPath()).setNewPath(srcPath.getPath()).build());
        throw t;
    }
    Metrics.PATHS_RENAMED.inc();
}