Examples with BlockMasterClient alluxio.worker.block.BlockMasterClient used on opensource projects

Example 1 with BlockMasterClient

use of alluxio.worker.block.BlockMasterClient in project alluxio by Alluxio.

the class RegisterWorkerBench method prepare.

@Override
public void prepare() throws Exception {
    // The task ID is different for local and cluster executions
    // So including that in the log can help associate the log to the run
    LOG.info("Task ID is {}", mBaseParameters.mId);
    mTierAliases = getTierAliases(mParameters.mTiers);
    mCapacityMap = Maps.toMap(mTierAliases, (tier) -> CAPACITY);
    mUsedMap = Maps.toMap(mTierAliases, (tier) -> 0L);
    // Generate block IDs heuristically
    Map<BlockStoreLocation, List<Long>> blockMap = RpcBenchPreparationUtils.generateBlockIdOnTiers(mParameters.mTiers);
    BlockMasterClient client = new BlockMasterClient(MasterClientContext.newBuilder(ClientContext.create(mConf)).build());
    mLocationBlockIdList = client.convertBlockListMapToProto(blockMap);
    // done once, so skip preparation when running in job worker
    if (!mBaseParameters.mDistributed) {
        // Prepare these block IDs concurrently
        LOG.info("Preparing blocks at the master");
        RpcBenchPreparationUtils.prepareBlocksInMaster(blockMap);
        LOG.info("Created all blocks at the master");
    }
    // Prepare worker IDs
    int numWorkers = mParameters.mConcurrency;
    mWorkerPool = RpcBenchPreparationUtils.prepareWorkerIds(client, numWorkers);
    Preconditions.checkState(mWorkerPool.size() == numWorkers, "Expecting %s workers but registered %s", numWorkers, mWorkerPool.size());
    LOG.info("Prepared worker IDs: {}", mWorkerPool);
}

Example 2 with BlockMasterClient

use of alluxio.worker.block.BlockMasterClient in project alluxio by Alluxio.

the class WorkerHeartbeatBench method prepare.

@Override
public void prepare() throws Exception {
    // The task ID is different for local and cluster executions
    // So including that in the log can help associate the log to the run
    LOG.info("Task ID is {}", mBaseParameters.mId);
    // Prepare block IDs to use for this test
    // We prepare the IDs before test starts so each RPC does not waste time in the conversion
    Map<BlockStoreLocation, List<Long>> blockMap = RpcBenchPreparationUtils.generateBlockIdOnTiers(mParameters.mTiers);
    BlockMasterClient client = new BlockMasterClient(MasterClientContext.newBuilder(ClientContext.create(mConf)).build());
    mLocationBlockIdList = client.convertBlockListMapToProto(blockMap);
    // done once, so skip preparation when running in job worker
    if (!mBaseParameters.mDistributed) {
        // Prepare these block IDs concurrently
        LOG.info("Preparing block IDs at the master");
        RpcBenchPreparationUtils.prepareBlocksInMaster(blockMap);
        LOG.info("Created all blocks at the master");
    }
    // Prepare simulated workers
    int numWorkers = mParameters.mConcurrency;
    LOG.info("Register {} simulated workers for the test", numWorkers);
    mWorkerPool = RpcBenchPreparationUtils.prepareWorkerIds(client, numWorkers);
    Preconditions.checkState(mWorkerPool.size() == numWorkers, "Expecting %s workers but registered %s", numWorkers, mWorkerPool.size());
    RpcBenchPreparationUtils.registerWorkers(client, mWorkerPool);
    LOG.info("All workers registered with the master {}", mWorkerPool);
}

Example 3 with BlockMasterClient

use of alluxio.worker.block.BlockMasterClient in project alluxio by Alluxio.

the class BlockWorkerDataReaderTest method before.

@Before
public void before() throws Exception {
    BlockMasterClient blockMasterClient = mock(BlockMasterClient.class);
    BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool());
    when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient);
    TieredBlockStore blockStore = new TieredBlockStore();
    FileSystemMasterClient fileSystemMasterClient = mock(FileSystemMasterClient.class);
    Sessions sessions = mock(Sessions.class);
    // Connect to the real UFS for UFS read testing
    UfsManager ufsManager = mock(UfsManager.class);
    mRootUfs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> UnderFileSystem.Factory.create(mRootUfs, UnderFileSystemConfiguration.defaults(ServerConfiguration.global())), new AlluxioURI(mRootUfs));
    when(ufsManager.get(anyLong())).thenReturn(ufsClient);
    mBlockWorker = new DefaultBlockWorker(blockMasterClientPool, fileSystemMasterClient, sessions, blockStore, ufsManager);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setBlockIds(Collections.singletonList(BLOCK_ID)));
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(mConf), mConf);
    mDataReaderFactory = new BlockWorkerDataReader.Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options);
}

Example 4 with BlockMasterClient

use of alluxio.worker.block.BlockMasterClient in project alluxio by Alluxio.

the class StreamRegisterWorkerBench method prepare.

@Override
public void prepare() throws Exception {
    // The task ID is different for local and cluster executions
    // So including that in the log can help associate the log to the run
    LOG.info("Task ID is {}", mBaseParameters.mId);
    mTierAliases = getTierAliases(mParameters.mTiers);
    mCapacityMap = Maps.toMap(mTierAliases, (tier) -> CAPACITY);
    mUsedMap = Maps.toMap(mTierAliases, (tier) -> 0L);
    // Generate block IDs heuristically
    Map<BlockStoreLocation, List<Long>> blockMap = RpcBenchPreparationUtils.generateBlockIdOnTiers(mParameters.mTiers);
    BlockMasterClient client = new BlockMasterClient(MasterClientContext.newBuilder(ClientContext.create(mConf)).build());
    mBlockMap = blockMap;
    // done once, so skip preparation when running in job worker
    if (!mBaseParameters.mDistributed) {
        // Prepare these block IDs concurrently
        LOG.info("Preparing blocks at the master");
        RpcBenchPreparationUtils.prepareBlocksInMaster(blockMap);
        LOG.info("Created all blocks at the master");
    }
    // Prepare worker IDs
    int numWorkers = mParameters.mConcurrency;
    mWorkerPool = RpcBenchPreparationUtils.prepareWorkerIds(client, numWorkers);
    Preconditions.checkState(mWorkerPool.size() == numWorkers, "Expecting %s workers but registered %s", numWorkers, mWorkerPool.size());
    LOG.info("Prepared worker IDs: {}", mWorkerPool);
}

Example 5 with BlockMasterClient

use of alluxio.worker.block.BlockMasterClient in project alluxio by Alluxio.

the class RpcBenchPreparationUtils method prepareBlocksInMaster.

/**
 * Prepare all relevant block IDs on the master side concurrently.
 *
 * @param locToBlocks a map from block location to block IDs
 */
public static void prepareBlocksInMaster(Map<BlockStoreLocation, List<Long>> locToBlocks) throws InterruptedException {
    // Since the task is I/O bound, set concurrency set to 4x CPUs
    int concurrency = Runtime.getRuntime().availableProcessors() * 4;
    // Partition the wanted block IDs to smaller jobs in order to utilize concurrency
    List<List<Long>> jobs = new ArrayList<>();
    long totalBlocks = 0;
    for (Map.Entry<BlockStoreLocation, List<Long>> e : locToBlocks.entrySet()) {
        List<Long> v = e.getValue();
        totalBlocks += v.size();
        jobs.addAll(Lists.partition(v, Math.min(v.size() / concurrency, 1_000)));
    }
    final long totalBlocksFinal = totalBlocks;
    LOG.info("Split block ID generation into {} jobs", jobs.size());
    for (List<Long> job : jobs) {
        LOG.debug("Block ids: [{},{}]", job.get(0), job.get(job.size() - 1));
    }
    ExecutorService pool = ExecutorServiceFactories.fixedThreadPool("rpc-bench-prepare", concurrency).create();
    long blockSize = sConf.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT);
    CompletableFuture[] futures = new CompletableFuture[jobs.size()];
    AtomicInteger progress = new AtomicInteger(0);
    for (int i = 0; i < jobs.size(); i++) {
        List<Long> job = jobs.get(i);
        final int batchIndex = i;
        final int batchSize = job.size();
        CompletableFuture<Void> future = CompletableFuture.supplyAsync((Supplier<Void>) () -> {
            BlockMasterClient client = new BlockMasterClient(MasterClientContext.newBuilder(ClientContext.create(sConf)).build());
            for (Long blockId : job) {
                try {
                    client.commitBlockInUfs(blockId, blockSize);
                } catch (IOException e) {
                    LOG.error("Failed to commitBlockInUfs in batch {}, blockId={} total={}", batchIndex, blockId, totalBlocksFinal, e);
                }
            }
            long finishedCount = progress.addAndGet(batchSize);
            LOG.info("Generated {}th batch of {} blocks, {}% completed", batchIndex, batchSize, String.format("%.2f", 100.0 * finishedCount / totalBlocksFinal));
            return null;
        }, pool);
        futures[i] = (future);
    }
    LOG.info("Collect all results");
    try {
        CompletableFuture.allOf(futures).join();
    } finally {
        pool.shutdownNow();
        pool.awaitTermination(30, TimeUnit.SECONDS);
    }
}