Examples with CloseableResource alluxio.resource.CloseableResource used on opensource projects

Example 1 with CloseableResource

use of alluxio.resource.CloseableResource in project alluxio by Alluxio.

the class DefaultFileSystemMaster method getRootMountInfo.

private static MountInfo getRootMountInfo(MasterUfsManager ufsManager) {
    try (CloseableResource<UnderFileSystem> resource = ufsManager.getRoot().acquireUfsResource()) {
        boolean shared = resource.get().isObjectStorage() && ServerConfiguration.getBoolean(PropertyKey.UNDERFS_OBJECT_STORE_MOUNT_SHARED_PUBLICLY);
        boolean readonly = ServerConfiguration.getBoolean(PropertyKey.MASTER_MOUNT_TABLE_ROOT_READONLY);
        String rootUfsUri = PathUtils.normalizePath(ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS), AlluxioURI.SEPARATOR);
        Map<String, String> rootUfsConf = ServerConfiguration.getNestedProperties(PropertyKey.MASTER_MOUNT_TABLE_ROOT_OPTION).entrySet().stream().filter(entry -> entry.getValue() != null).collect(Collectors.toMap(Map.Entry::getKey, entry -> String.valueOf(entry.getValue())));
        MountPOptions mountOptions = MountContext.mergeFrom(MountPOptions.newBuilder().setShared(shared).setReadOnly(readonly).putAllProperties(rootUfsConf)).getOptions().build();
        return new MountInfo(new AlluxioURI(MountTable.ROOT), new AlluxioURI(rootUfsUri), IdUtils.ROOT_MOUNT_ID, mountOptions);
    }
}

Example 2 with CloseableResource

use of alluxio.resource.CloseableResource in project alluxio by Alluxio.

the class FileSystemContext method acquireBlockWorkerClientInternal.

private CloseableResource<BlockWorkerClient> acquireBlockWorkerClientInternal(final WorkerNetAddress workerNetAddress, final ClientContext context, UserState userState) throws IOException {
    SocketAddress address = NetworkAddressUtils.getDataPortSocketAddress(workerNetAddress, context.getClusterConf());
    GrpcServerAddress serverAddress = GrpcServerAddress.create(workerNetAddress.getHost(), address);
    ClientPoolKey key = new ClientPoolKey(address, AuthenticationUserUtils.getImpersonationUser(userState.getSubject(), context.getClusterConf()));
    final ConcurrentHashMap<ClientPoolKey, BlockWorkerClientPool> poolMap = mBlockWorkerClientPoolMap;
    return new CloseableResource<BlockWorkerClient>(poolMap.computeIfAbsent(key, k -> new BlockWorkerClientPool(userState, serverAddress, context.getClusterConf().getInt(PropertyKey.USER_BLOCK_WORKER_CLIENT_POOL_MIN), context.getClusterConf().getInt(PropertyKey.USER_BLOCK_WORKER_CLIENT_POOL_MAX), context.getClusterConf())).acquire()) {

        // Save the reference to the original pool map.
        @Override
        public void closeResource() {
            releaseBlockWorkerClient(workerNetAddress, get(), context, poolMap);
        }
    };
}

Example 3 with CloseableResource

use of alluxio.resource.CloseableResource in project alluxio by Alluxio.

the class ActiveSyncManager method start.

/**
 * Start the polling threads.
 */
public void start() throws IOException {
    mStarted = true;
    // Initialize UFS states
    for (AlluxioURI syncPoint : mSyncPathList) {
        MountTable.Resolution resolution;
        try {
            resolution = mMountTable.resolve(syncPoint);
        } catch (InvalidPathException e) {
            LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " + "path {}, exception {}", syncPoint, e);
            continue;
        }
        try (CloseableResource<UnderFileSystem> ufsResource = resolution.acquireUfsResource()) {
            if (!ufsResource.get().supportsActiveSync()) {
                throw new UnsupportedOperationException("Active Sync is not supported on this UFS type: " + ufsResource.get().getUnderFSType());
            }
            ufsResource.get().startSync(resolution.getUri());
        }
    }
    // therefore forces a sync
    for (Map.Entry<Long, List<AlluxioURI>> entry : mFilterMap.entrySet()) {
        long mountId = entry.getKey();
        long txId = mStartingTxIdMap.getOrDefault(mountId, SyncInfo.INVALID_TXID);
        if (!entry.getValue().isEmpty()) {
            launchPollingThread(mountId, txId);
        }
        try {
            if ((txId == SyncInfo.INVALID_TXID) && ServerConfiguration.getBoolean(PropertyKey.MASTER_UFS_ACTIVE_SYNC_INITIAL_SYNC_ENABLED)) {
                mExecutorService.submit(() -> entry.getValue().parallelStream().forEach(syncPoint -> {
                    MountTable.Resolution resolution;
                    try {
                        resolution = mMountTable.resolve(syncPoint);
                    } catch (InvalidPathException e) {
                        LOG.info("Invalid Path encountered during start up of ActiveSyncManager, " + "path {}, exception {}", syncPoint, e);
                        return;
                    }
                    startInitialFullSync(syncPoint, resolution);
                }));
            }
        } catch (Exception e) {
            LOG.warn("exception encountered during initial sync: {}", e.toString());
        }
    }
}

Example 4 with CloseableResource

use of alluxio.resource.CloseableResource in project alluxio by Alluxio.

the class GetPinnedFileIdsBench method prepare.

@Override
public void prepare() throws Exception {
    // The task ID is different for local and cluster executions
    // So including that in the log can help associate the log to the run
    LOG.info("Task ID is {}", mBaseParameters.mId);
    // done once, so skip preparation when running in job worker
    if (mBaseParameters.mDistributed) {
        LOG.info("Skipping preparation in distributed execution");
        return;
    }
    AlluxioURI baseUri = new AlluxioURI(mParameters.mBasePath);
    try (CloseableResource<alluxio.client.file.FileSystemMasterClient> client = mFileSystemContext.acquireMasterClientResource()) {
        LOG.info("Creating temporary directory {} for benchmark", baseUri);
        client.get().createDirectory(baseUri, CreateDirectoryPOptions.newBuilder().setAllowExists(true).build());
    }
    int numFiles = mParameters.mNumFiles;
    int fileNameLength = (int) Math.ceil(Math.max(8, Math.log10(numFiles)));
    CompletableFuture<Void>[] futures = new CompletableFuture[numFiles];
    LOG.info("Generating {} pinned test files at the master", numFiles);
    for (int i = 0; i < numFiles; i++) {
        AlluxioURI fileUri = baseUri.join(CommonUtils.randomAlphaNumString(fileNameLength));
        CompletableFuture<Void> future = CompletableFuture.supplyAsync((Supplier<Void>) () -> {
            try (CloseableResource<FileSystemMasterClient> client = mFileSystemContext.acquireMasterClientResource()) {
                client.get().createFile(fileUri, CreateFilePOptions.newBuilder().setBlockSizeBytes(mConf.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT)).build());
                client.get().setAttribute(fileUri, SetAttributePOptions.newBuilder().setPinned(true).build());
            } catch (AlluxioStatusException e) {
                LOG.warn("Exception during file creation of {}", fileUri, e);
            }
            return null;
        }, getPool());
        futures[i] = (future);
    }
    CompletableFuture.allOf(futures).join();
    LOG.info("Test files generated");
}

Example 5 with CloseableResource

use of alluxio.resource.CloseableResource in project alluxio by Alluxio.

the class HiveDatabase method getTable.

@Override
public UdbTable getTable(String tableName, UdbBypassSpec bypassSpec) throws IOException {
    try {
        Table table;
        List<Partition> partitions;
        List<ColumnStatisticsObj> columnStats;
        List<String> partitionColumns;
        Map<String, List<ColumnStatisticsInfo>> statsMap = new HashMap<>();
        // perform all the hive client operations, and release the client early.
        try (CloseableResource<IMetaStoreClient> client = mClientPool.acquireClientResource()) {
            table = client.get().getTable(mHiveDbName, tableName);
            // Potentially expensive call
            partitions = client.get().listPartitions(mHiveDbName, table.getTableName(), (short) -1);
            List<String> colNames = table.getSd().getCols().stream().map(FieldSchema::getName).collect(Collectors.toList());
            columnStats = client.get().getTableColumnStatistics(mHiveDbName, tableName, colNames);
            // construct the partition statistics
            List<String> dataColumns = table.getSd().getCols().stream().map(org.apache.hadoop.hive.metastore.api.FieldSchema::getName).collect(Collectors.toList());
            partitionColumns = table.getPartitionKeys().stream().map(org.apache.hadoop.hive.metastore.api.FieldSchema::getName).collect(Collectors.toList());
            List<String> partitionNames = partitions.stream().map(partition -> FileUtils.makePartName(partitionColumns, partition.getValues())).collect(Collectors.toList());
            for (List<String> partialPartitionNames : Lists.partition(partitionNames, MAX_PARTITION_COLUMN_STATISTICS)) {
                statsMap.putAll(client.get().getPartitionColumnStatistics(mHiveDbName, tableName, partialPartitionNames, dataColumns).entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().stream().map(HiveUtils::toProto).collect(Collectors.toList()), (e1, e2) -> e2)));
            }
        }
        PathTranslator pathTranslator = mountAlluxioPaths(table, partitions, bypassSpec);
        List<ColumnStatisticsInfo> colStats = columnStats.stream().map(HiveUtils::toProto).collect(Collectors.toList());
        // construct table layout
        PartitionInfo partitionInfo = PartitionInfo.newBuilder().setDbName(getUdbContext().getDbName()).setTableName(tableName).addAllDataCols(HiveUtils.toProto(table.getSd().getCols())).setStorage(HiveUtils.toProto(table.getSd(), pathTranslator)).putAllParameters(table.getParameters()).build();
        Layout layout = Layout.newBuilder().setLayoutType(HiveLayout.TYPE).setLayoutData(partitionInfo.toByteString()).build();
        // create udb partitions info
        List<UdbPartition> udbPartitions = new ArrayList<>();
        if (partitionColumns.isEmpty()) {
            // unpartitioned table, generate a partition
            PartitionInfo.Builder pib = PartitionInfo.newBuilder().setDbName(getUdbContext().getDbName()).setTableName(tableName).addAllDataCols(HiveUtils.toProto(table.getSd().getCols())).setStorage(HiveUtils.toProto(table.getSd(), pathTranslator)).setPartitionName(tableName).putAllParameters(table.getParameters());
            udbPartitions.add(new HivePartition(new HiveLayout(pib.build(), Collections.emptyList())));
        } else {
            for (Partition partition : partitions) {
                String partName = FileUtils.makePartName(partitionColumns, partition.getValues());
                PartitionInfo.Builder pib = PartitionInfo.newBuilder().setDbName(getUdbContext().getDbName()).setTableName(tableName).addAllDataCols(HiveUtils.toProto(partition.getSd().getCols())).setStorage(HiveUtils.toProto(partition.getSd(), pathTranslator)).setPartitionName(partName).putAllParameters(partition.getParameters());
                if (partition.getValues() != null) {
                    pib.addAllValues(partition.getValues());
                }
                udbPartitions.add(new HivePartition(new HiveLayout(pib.build(), statsMap.getOrDefault(partName, Collections.emptyList()))));
            }
        }
        return new HiveTable(tableName, HiveUtils.toProtoSchema(table.getSd().getCols()), colStats, HiveUtils.toProto(table.getPartitionKeys()), udbPartitions, layout, table);
    } catch (NoSuchObjectException e) {
        throw new NotFoundException("Table " + tableName + " does not exist.", e);
    } catch (TException e) {
        throw new IOException("Failed to get table: " + tableName + " error: " + e.getMessage(), e);
    }
}