Examples with LocationBlockIdListEntry alluxio.grpc.LocationBlockIdListEntry used on opensource projects

Example 11 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class RegisterStreamTestUtils method findFirstBlock.

static long findFirstBlock(List<RegisterWorkerPRequest> chunks) {
    RegisterWorkerPRequest firstBatch = chunks.get(0);
    LocationBlockIdListEntry entry = firstBatch.getCurrentBlocks(0);
    return entry.getValue().getBlockId(0);
}

Example 12 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class StreamRegisterWorkerBench method debriefBlockListProto.

private static void debriefBlockListProto(List<LocationBlockIdListEntry> entries) {
    StringBuilder sb = new StringBuilder();
    for (LocationBlockIdListEntry e : entries) {
        sb.append(String.format("%s,", e.getKey()));
    }
    LOG.info("Generated locations: {}", sb.toString());
}

Example 13 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class RegisterWorkerBench method runOnce.

private void runOnce(alluxio.worker.block.BlockMasterClient client, RpcTaskResult result, long i, long workerId) {
    try {
        Instant s = Instant.now();
        if (mConf.getBoolean(PropertyKey.WORKER_REGISTER_LEASE_ENABLED)) {
            LOG.info("Acquiring lease for {}", workerId);
            int blockCount = 0;
            for (LocationBlockIdListEntry entry : mLocationBlockIdList) {
                blockCount += entry.getValue().getBlockIdCount();
            }
            client.acquireRegisterLeaseWithBackoff(workerId, blockCount, BlockMasterSync.getDefaultAcquireLeaseRetryPolicy());
            LOG.info("Lease acquired for {}", workerId);
        }
        // TODO(jiacheng): The 1st reported RPC time is always very long, this does
        // not match with the time recorded by Jaeger.
        // I suspect it's the time spend in establishing the connection.
        // The easiest way out is just to ignore the 1st point.
        client.register(workerId, mTierAliases, mCapacityMap, mUsedMap, // So an empty block list will be used here
        ImmutableMap.of(), // lost storage
        LOST_STORAGE, // extra config
        EMPTY_CONFIG);
        LOG.info("Worker {} registered", workerId);
        Instant e = Instant.now();
        RpcTaskResult.Point p = new RpcTaskResult.Point(Duration.between(s, e).toMillis());
        result.addPoint(p);
        LOG.debug("Iter {} took {}ns", i, p.mDurationMs);
    } catch (Exception e) {
        LOG.error("Failed to run iter {}", i, e);
        result.addError(e.getMessage());
    }
}

Example 14 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMasterClient method register.

/**
 * The method the worker should execute to register with the block master.
 *
 * @param workerId the worker id of the worker registering
 * @param storageTierAliases a list of storage tier aliases in ordinal order
 * @param totalBytesOnTiers mapping from storage tier alias to total bytes
 * @param usedBytesOnTiers mapping from storage tier alias to used bytes
 * @param currentBlocksOnLocation mapping from storage tier alias to the list of list of blocks
 * @param lostStorage mapping from storage tier alias to the list of lost storage paths
 * @param configList a list of configurations
 */
// TODO(yupeng): rename to workerBlockReport or workerInitialize?
public void register(final long workerId, final List<String> storageTierAliases, final Map<String, Long> totalBytesOnTiers, final Map<String, Long> usedBytesOnTiers, final Map<BlockStoreLocation, List<Long>> currentBlocksOnLocation, final Map<String, List<String>> lostStorage, final List<ConfigProperty> configList) throws IOException {
    final RegisterWorkerPOptions options = RegisterWorkerPOptions.newBuilder().addAllConfigs(configList).build();
    final List<LocationBlockIdListEntry> currentBlocks = convertBlockListMapToProto(currentBlocksOnLocation);
    final Map<String, StorageList> lostStorageMap = lostStorage.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> StorageList.newBuilder().addAllStorage(e.getValue()).build()));
    final RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addAllStorageTiers(storageTierAliases).putAllTotalBytesOnTiers(totalBytesOnTiers).putAllUsedBytesOnTiers(usedBytesOnTiers).addAllCurrentBlocks(currentBlocks).putAllLostStorage(lostStorageMap).setOptions(options).build();
    retryRPC(() -> {
        mClient.registerWorker(request);
        return null;
    }, LOG, "Register", "workerId=%d", workerId);
}

Example 15 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMapIterator method nextBatchFromTier.

private LocationBlockIdListEntry nextBatchFromTier(BlockStoreLocationProto currentLoc, Iterator<Long> currentIterator, int spaceLeft) {
    // Generate the next batch
    List<Long> blockIdBatch = new ArrayList<>(spaceLeft);
    while (blockIdBatch.size() < spaceLeft && currentIterator.hasNext()) {
        blockIdBatch.add(currentIterator.next());
        mCounter++;
    }
    BlockIdList blockIdList = BlockIdList.newBuilder().addAllBlockId(blockIdBatch).build();
    LocationBlockIdListEntry listEntry = LocationBlockIdListEntry.newBuilder().setKey(currentLoc).setValue(blockIdList).build();
    return listEntry;
}