Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 31 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getRootMountInfo.

private static MountInfo getRootMountInfo(MasterUfsManager ufsManager) {
    try (CloseableResource<UnderFileSystem> resource = ufsManager.getRoot().acquireUfsResource()) {
        boolean shared = resource.get().isObjectStorage() && ServerConfiguration.getBoolean(PropertyKey.UNDERFS_OBJECT_STORE_MOUNT_SHARED_PUBLICLY);
        boolean readonly = ServerConfiguration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_READONLY);
        String rootUfsUri = PathUtils.normalizePath(ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS), AlluxioURI.SEPARATOR);
        Map<String, String> rootUfsConf = ServerConfiguration.getNestedProperties(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION).entrySet().stream().filter(entry -> entry.getValue() != null).collect(Collectors.toMap(Map.Entry::getKey, entry -> String.valueOf(entry.getValue())));
        MountPOptions mountOptions = MountContext.mergeFrom(MountPOptions.newBuilder().setShared(shared).setReadOnly(readonly).putAllProperties(rootUfsConf)).getOptions().build();
        return new MountInfo(new AlluxioURI(MountTable.ROOT), new AlluxioURI(rootUfsUri), IdUtils.ROOT_MOUNT_ID, mountOptions);
    }
}

Example 32 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class LazyUfsBlockLocationCache method get.

@Override
@Nullable
public List<String> get(long blockId, AlluxioURI fileUri, long offset) {
    List<String> locations = mCache.getIfPresent(blockId);
    if (locations != null) {
        return locations;
    }
    try {
        MountTable.Resolution resolution = mMountTable.resolve(fileUri);
        String ufsUri = resolution.getUri().toString();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            UnderFileSystem ufs = ufsResource.get();
            locations = ufs.getFileLocations(ufsUri, FileLocationOptions.defaults().setOffset(offset));
        }
        if (locations != null) {
            mCache.put(blockId, locations);
            return locations;
        }
    } catch (InvalidPathException | IOException e) {
        LOG.warn("Failed to get locations for block {} in file {} with offset {}: {}", blockId, fileUri, offset, e);
    }
    return null;
}

Example 33 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class ActiveSyncManager method start.

/**
 * Start the polling threads.
 */
public void start() throws IOException {
    mStarted = true;
    // Initialize UFS states
    for (AlluxioURI syncPoint : mSyncPathList) {
        MountTable.Resolution resolution;
        try {
            resolution = mMountTable.resolve(syncPoint);
        } catch (InvalidPathException e) {
            LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " + "path {}, exception {}", syncPoint, e);
            continue;
        }
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            if (!ufsResource.get().supportsActiveSync()) {
                throw new UnsupportedOperationException("Active Sync is not supported on this UFS type: " + ufsResource.get().getUnderFSType());
            }
            ufsResource.get().startSync(resolution.getUri());
        }
    }
    // therefore forces a sync
    for (Map.Entry<Long, List<AlluxioURI>> entry : mFilterMap.entrySet()) {
        long mountId = entry.getKey();
        long txId = mStartingTxIdMap.getOrDefault(mountId, SyncInfo.INVALID_TXID);
        if (!entry.getValue().isEmpty()) {
            launchPollingThread(mountId, txId);
        }
        try {
            if ((txId == SyncInfo.INVALID_TXID) && ServerConfiguration.getBoolean(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INITIAL_SYNC_ENABLED)) {
                mExecutorService.submit(() -> entry.getValue().parallelStream().forEach(syncPoint -> {
                    MountTable.Resolution resolution;
                    try {
                        resolution = mMountTable.resolve(syncPoint);
                    } catch (InvalidPathException e) {
                        LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " + "path {}, exception {}", syncPoint, e);
                        return;
                    }
                    startInitialFullSync(syncPoint, resolution);
                }));
            }
        } catch (Exception e) {
            LOG.warn("exception encountered during initial sync: {}", e.toString());
        }
    }
}

Example 34 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class ActiveSyncManager method launchPollingThread.

/**
 * Launches polling thread on a particular mount point with starting txId.
 *
 * @param mountId launch polling thread on a mount id
 * @param txId specifies the transaction id to initialize the pollling thread
 */
public void launchPollingThread(long mountId, long txId) {
    LOG.debug("launch polling thread for mount id {}, txId {}", mountId, txId);
    if (!mPollerMap.containsKey(mountId)) {
        UfsManager.UfsClient ufsClient = mMountTable.getUfsClient(mountId);
        if (ufsClient == null) {
            LOG.warn("Mount id {} does not exist", mountId);
            return;
        }
        try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) {
            ufsResource.get().startActiveSyncPolling(txId);
        } catch (IOException e) {
            LOG.warn("IO Exception trying to launch Polling thread: {}", e.toString());
        }
        ActiveSyncer syncer = new ActiveSyncer(mFileSystemMaster, this, mMountTable, mountId);
        Future<?> future = getExecutor().submit(new HeartbeatThread(HeartbeatContext.MASTER_ACTIVE_UFS_SYNC, syncer, (int) ServerConfiguration.getMs(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INTERVAL), ServerConfiguration.global(), ServerUserState.global()));
        mPollerMap.put(mountId, future);
    }
}

Example 35 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class ActiveSyncManager method startSyncAndJournal.

/**
 * Start active sync on a URI and journal the add entry.
 *
 * @param rpcContext the master rpc or no-op context
 * @param syncPoint sync point to be start
 */
public void startSyncAndJournal(RpcContext rpcContext, AlluxioURI syncPoint) throws InvalidPathException {
    try (LockResource r = new LockResource(mLock)) {
        MountTable.Resolution resolution = mMountTable.resolve(syncPoint);
        long mountId = resolution.getMountId();
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            if (!ufsResource.get().supportsActiveSync()) {
                throw new UnsupportedOperationException("Active Syncing is not supported on this UFS type: " + ufsResource.get().getUnderFSType());
            }
        }
        if (isUnderSyncPoint(syncPoint)) {
            throw new InvalidPathException("URI " + syncPoint + " is already a sync point");
        }
        AddSyncPointEntry addSyncPoint = AddSyncPointEntry.newBuilder().setSyncpointPath(syncPoint.toString()).setMountId(mountId).build();
        applyAndJournal(rpcContext, addSyncPoint);
        try {
            startSyncInternal(syncPoint, resolution);
        } catch (Throwable e) {
            LOG.warn("Start sync failed on {}", syncPoint, e);
            // revert state;
            RemoveSyncPointEntry removeSyncPoint = File.RemoveSyncPointEntry.newBuilder().setSyncpointPath(syncPoint.toString()).build();
            applyAndJournal(rpcContext, removeSyncPoint);
            recoverFromStartSync(syncPoint, resolution.getMountId());
            throw e;
        }
    }
}