Examples with LocationBlockIdListEntry alluxio.grpc.LocationBlockIdListEntry used on opensource projects

Example 6 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMasterClient method heartbeat.

/**
 * The method the worker should periodically execute to heartbeat back to the master.
 *
 * @param workerId the worker id
 * @param capacityBytesOnTiers a mapping from storage tier alias to capacity bytes
 * @param usedBytesOnTiers a mapping from storage tier alias to used bytes
 * @param removedBlocks a list of block removed from this worker
 * @param addedBlocks a mapping from storage tier alias to added blocks
 * @param lostStorage a mapping from storage tier alias to a list of lost storage paths
 * @param metrics a list of worker metrics
 * @return an optional command for the worker to execute
 */
public synchronized Command heartbeat(final long workerId, final Map<String, Long> capacityBytesOnTiers, final Map<String, Long> usedBytesOnTiers, final List<Long> removedBlocks, final Map<BlockStoreLocation, List<Long>> addedBlocks, final Map<String, List<String>> lostStorage, final List<Metric> metrics) throws IOException {
    final BlockHeartbeatPOptions options = BlockHeartbeatPOptions.newBuilder().addAllMetrics(metrics).putAllCapacityBytesOnTiers(capacityBytesOnTiers).build();
    final List<LocationBlockIdListEntry> entryList = convertBlockListMapToProto(addedBlocks);
    final Map<String, StorageList> lostStorageMap = lostStorage.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> StorageList.newBuilder().addAllStorage(e.getValue()).build()));
    final BlockHeartbeatPRequest request = BlockHeartbeatPRequest.newBuilder().setWorkerId(workerId).putAllUsedBytesOnTiers(usedBytesOnTiers).addAllRemovedBlockIds(removedBlocks).addAllAddedBlocks(entryList).setOptions(options).putAllLostStorage(lostStorageMap).build();
    return retryRPC(() -> mClient.withDeadlineAfter(mContext.getClusterConf().getMs(PropertyKey.WORKER_MASTER_PERIODICAL_RPC_TIMEOUT), TimeUnit.MILLISECONDS).blockHeartbeat(request).getCommand(), LOG, "Heartbeat", "workerId=%d", workerId);
}

Example 7 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMasterClient method convertBlockListMapToProto.

/**
 * Converts the block list map to a proto list.
 * Because the list is flattened from a map, in the list no two {@link LocationBlockIdListEntry}
 * instances shall have the same {@link BlockStoreLocationProto}.
 * The uniqueness of {@link BlockStoreLocationProto} is determined by tier alias and medium type.
 * That means directories with the same tier alias and medium type will be merged into the same
 * {@link LocationBlockIdListEntry}.
 *
 * @param blockListOnLocation a map from block location to the block list
 * @return a flattened and deduplicated list
 */
@VisibleForTesting
public List<LocationBlockIdListEntry> convertBlockListMapToProto(Map<BlockStoreLocation, List<Long>> blockListOnLocation) {
    final List<LocationBlockIdListEntry> entryList = new ArrayList<>();
    Map<BlockStoreLocationProto, List<Long>> tierToBlocks = new HashMap<>();
    for (Map.Entry<BlockStoreLocation, List<Long>> entry : blockListOnLocation.entrySet()) {
        BlockStoreLocation loc = entry.getKey();
        BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder().setTierAlias(loc.tierAlias()).setMediumType(loc.mediumType()).build();
        if (tierToBlocks.containsKey(locationProto)) {
            tierToBlocks.get(locationProto).addAll(entry.getValue());
        } else {
            List<Long> blockList = new ArrayList<>(entry.getValue());
            tierToBlocks.put(locationProto, blockList);
        }
    }
    for (Map.Entry<BlockStoreLocationProto, List<Long>> entry : tierToBlocks.entrySet()) {
        BlockIdList blockIdList = BlockIdList.newBuilder().addAllBlockId(entry.getValue()).build();
        LocationBlockIdListEntry listEntry = LocationBlockIdListEntry.newBuilder().setKey(entry.getKey()).setValue(blockIdList).build();
        entryList.add(listEntry);
    }
    return entryList;
}

Example 8 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class RegisterStreamer method next.

@Override
public RegisterWorkerPRequest next() {
    RegisterWorkerPRequest request;
    List<LocationBlockIdListEntry> blockBatch;
    if (mBatchNumber == 0) {
        if (mBlockMapIterator.hasNext()) {
            blockBatch = mBlockMapIterator.next();
        } else {
            blockBatch = Collections.emptyList();
        }
        // If it is the 1st batch, include metadata
        request = RegisterWorkerPRequest.newBuilder().setWorkerId(mWorkerId).addAllStorageTiers(mStorageTierAliases).putAllTotalBytesOnTiers(mTotalBytesOnTiers).putAllUsedBytesOnTiers(mUsedBytesOnTiers).putAllLostStorage(mLostStorageMap).setOptions(mOptions).addAllCurrentBlocks(blockBatch).build();
    } else {
        blockBatch = mBlockMapIterator.next();
        // Following batches only include the block list
        request = RegisterWorkerPRequest.newBuilder().setWorkerId(mWorkerId).addAllCurrentBlocks(blockBatch).build();
    }
    mBatchNumber++;
    return request;
}

Example 9 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMasterClientTest method convertBlockListMapToProtoMergeDirsInSameTier.

@Test
public void convertBlockListMapToProtoMergeDirsInSameTier() {
    BlockMasterClient client = new BlockMasterClient(MasterClientContext.newBuilder(ClientContext.create(mConf)).build());
    Map<BlockStoreLocation, List<Long>> blockMap = new HashMap<>();
    BlockStoreLocation memDir0 = new BlockStoreLocation("MEM", 0);
    blockMap.put(memDir0, Arrays.asList(1L, 2L, 3L));
    BlockStoreLocation memDir1 = new BlockStoreLocation("MEM", 1);
    blockMap.put(memDir1, Arrays.asList(4L, 5L, 6L, 7L));
    BlockStoreLocation ssdDir0 = new BlockStoreLocation("SSD", 0);
    blockMap.put(ssdDir0, Arrays.asList(11L, 12L, 13L, 14L));
    BlockStoreLocation ssdDir1 = new BlockStoreLocation("SSD", 1);
    blockMap.put(ssdDir1, Arrays.asList(15L, 16L, 17L, 18L, 19L));
    // Directories on the same tier will be merged together
    List<LocationBlockIdListEntry> protoList = client.convertBlockListMapToProto(blockMap);
    assertEquals(2, protoList.size());
    BlockStoreLocationProto memLocationProto = BlockStoreLocationProto.newBuilder().setTierAlias("MEM").setMediumType("").build();
    BlockStoreLocationProto ssdLocationProto = BlockStoreLocationProto.newBuilder().setTierAlias("SSD").setMediumType("").build();
    Set<BlockStoreLocationProto> blockLocations = protoList.stream().map(LocationBlockIdListEntry::getKey).collect(Collectors.toSet());
    assertEquals(ImmutableSet.of(memLocationProto, ssdLocationProto), blockLocations);
    LocationBlockIdListEntry firstEntry = protoList.get(0);
    if (firstEntry.getKey().getTierAlias().equals("MEM")) {
        LocationBlockIdListEntry memTierEntry = protoList.get(0);
        List<Long> memProtoBlockList = memTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(1L, 2L, 3L, 4L, 5L, 6L, 7L), new HashSet<>(memProtoBlockList));
        LocationBlockIdListEntry ssdTierEntry = protoList.get(1);
        List<Long> ssdProtoBlockList = ssdTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L), new HashSet<>(ssdProtoBlockList));
    } else {
        LocationBlockIdListEntry memTierEntry = protoList.get(1);
        List<Long> memProtoBlockList = memTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(1L, 2L, 3L, 4L, 5L, 6L, 7L), new HashSet<>(memProtoBlockList));
        LocationBlockIdListEntry ssdTierEntry = protoList.get(0);
        List<Long> ssdProtoBlockList = ssdTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L), new HashSet<>(ssdProtoBlockList));
    }
}

Example 10 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method requestsForEmptyWorker.

/**
 * Tests below cover the most normal cases.
 */
@Test
public void requestsForEmptyWorker() throws Exception {
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForEmptyWorker(WORKER_ID);
    // Verify the size and content of the requests
    assertEquals(1, requestChunks.size());
    RegisterWorkerPRequest request = requestChunks.get(0);
    assertEquals(WORKER_ID, request.getWorkerId());
    assertEquals(MEM_USAGE_EMPTY, request.getUsedBytesOnTiersMap());
    assertEquals(MEM_CAPACITY, request.getTotalBytesOnTiersMap());
    Map<String, StorageList> lostMap = request.getLostStorageMap();
    assertEquals(1, lostMap.size());
    assertEquals(StorageList.newBuilder().build(), lostMap.get("MEM"));
    assertEquals(ImmutableList.of("MEM"), request.getStorageTiersList());
    List<LocationBlockIdListEntry> entries = request.getCurrentBlocksList();
    assertEquals(0, entries.size());
}