Examples with BlockStoreLocationProto alluxio.grpc.BlockStoreLocationProto used on opensource projects

Example 6 with BlockStoreLocationProto

use of alluxio.grpc.BlockStoreLocationProto in project alluxio by Alluxio.

the class BlockMasterClient method convertBlockListMapToProto.

/**
 * Converts the block list map to a proto list.
 * Because the list is flattened from a map, in the list no two {@link LocationBlockIdListEntry}
 * instances shall have the same {@link BlockStoreLocationProto}.
 * The uniqueness of {@link BlockStoreLocationProto} is determined by tier alias and medium type.
 * That means directories with the same tier alias and medium type will be merged into the same
 * {@link LocationBlockIdListEntry}.
 *
 * @param blockListOnLocation a map from block location to the block list
 * @return a flattened and deduplicated list
 */
@VisibleForTesting
public List<LocationBlockIdListEntry> convertBlockListMapToProto(Map<BlockStoreLocation, List<Long>> blockListOnLocation) {
    final List<LocationBlockIdListEntry> entryList = new ArrayList<>();
    Map<BlockStoreLocationProto, List<Long>> tierToBlocks = new HashMap<>();
    for (Map.Entry<BlockStoreLocation, List<Long>> entry : blockListOnLocation.entrySet()) {
        BlockStoreLocation loc = entry.getKey();
        BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder().setTierAlias(loc.tierAlias()).setMediumType(loc.mediumType()).build();
        if (tierToBlocks.containsKey(locationProto)) {
            tierToBlocks.get(locationProto).addAll(entry.getValue());
        } else {
            List<Long> blockList = new ArrayList<>(entry.getValue());
            tierToBlocks.put(locationProto, blockList);
        }
    }
    for (Map.Entry<BlockStoreLocationProto, List<Long>> entry : tierToBlocks.entrySet()) {
        BlockIdList blockIdList = BlockIdList.newBuilder().addAllBlockId(entry.getValue()).build();
        LocationBlockIdListEntry listEntry = LocationBlockIdListEntry.newBuilder().setKey(entry.getKey()).setValue(blockIdList).build();
        entryList.add(listEntry);
    }
    return entryList;
}

Example 7 with BlockStoreLocationProto

use of alluxio.grpc.BlockStoreLocationProto in project alluxio by Alluxio.

the class BlockMasterClientTest method convertBlockListMapToProtoMergeDirsInSameTier.

@Test
public void convertBlockListMapToProtoMergeDirsInSameTier() {
    BlockMasterClient client = new BlockMasterClient(MasterClientContext.newBuilder(ClientContext.create(mConf)).build());
    Map<BlockStoreLocation, List<Long>> blockMap = new HashMap<>();
    BlockStoreLocation memDir0 = new BlockStoreLocation("MEM", 0);
    blockMap.put(memDir0, Arrays.asList(1L, 2L, 3L));
    BlockStoreLocation memDir1 = new BlockStoreLocation("MEM", 1);
    blockMap.put(memDir1, Arrays.asList(4L, 5L, 6L, 7L));
    BlockStoreLocation ssdDir0 = new BlockStoreLocation("SSD", 0);
    blockMap.put(ssdDir0, Arrays.asList(11L, 12L, 13L, 14L));
    BlockStoreLocation ssdDir1 = new BlockStoreLocation("SSD", 1);
    blockMap.put(ssdDir1, Arrays.asList(15L, 16L, 17L, 18L, 19L));
    // Directories on the same tier will be merged together
    List<LocationBlockIdListEntry> protoList = client.convertBlockListMapToProto(blockMap);
    assertEquals(2, protoList.size());
    BlockStoreLocationProto memLocationProto = BlockStoreLocationProto.newBuilder().setTierAlias("MEM").setMediumType("").build();
    BlockStoreLocationProto ssdLocationProto = BlockStoreLocationProto.newBuilder().setTierAlias("SSD").setMediumType("").build();
    Set<BlockStoreLocationProto> blockLocations = protoList.stream().map(LocationBlockIdListEntry::getKey).collect(Collectors.toSet());
    assertEquals(ImmutableSet.of(memLocationProto, ssdLocationProto), blockLocations);
    LocationBlockIdListEntry firstEntry = protoList.get(0);
    if (firstEntry.getKey().getTierAlias().equals("MEM")) {
        LocationBlockIdListEntry memTierEntry = protoList.get(0);
        List<Long> memProtoBlockList = memTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(1L, 2L, 3L, 4L, 5L, 6L, 7L), new HashSet<>(memProtoBlockList));
        LocationBlockIdListEntry ssdTierEntry = protoList.get(1);
        List<Long> ssdProtoBlockList = ssdTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L), new HashSet<>(ssdProtoBlockList));
    } else {
        LocationBlockIdListEntry memTierEntry = protoList.get(1);
        List<Long> memProtoBlockList = memTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(1L, 2L, 3L, 4L, 5L, 6L, 7L), new HashSet<>(memProtoBlockList));
        LocationBlockIdListEntry ssdTierEntry = protoList.get(0);
        List<Long> ssdProtoBlockList = ssdTierEntry.getValue().getBlockIdList();
        assertEquals(ImmutableSet.of(11L, 12L, 13L, 14L, 15L, 16L, 17L, 18L, 19L), new HashSet<>(ssdProtoBlockList));
    }
}

Example 8 with BlockStoreLocationProto

use of alluxio.grpc.BlockStoreLocationProto in project alluxio by Alluxio.

the class BlockMapIterator method next.

@Override
public List<LocationBlockIdListEntry> next() {
    List<LocationBlockIdListEntry> result = new ArrayList<>();
    int currentCounter = mCounter;
    int targetCounter = currentCounter + mBatchSize;
    while (mCounter < targetCounter) {
        // Find the BlockStoreLocation
        BlockStoreLocationProto currentLoc = mBlockStoreLocationProtoList.get(mCurrentBlockLocationIndex);
        Iterator<Long> currentIterator = mBlockLocationIteratorMap.get(currentLoc);
        // Generate the next batch
        // This method does NOT progress the pointers
        int spaceLeft = targetCounter - mCounter;
        LocationBlockIdListEntry batchFromThisTier = nextBatchFromTier(currentLoc, currentIterator, spaceLeft);
        result.add(batchFromThisTier);
        // Progress the iterator based on the break condition
        if (!currentIterator.hasNext()) {
            // We keep filling in using the next tier
            mCurrentBlockLocationIndex++;
            if (mCurrentBlockLocationIndex >= mBlockStoreLocationProtoList.size()) {
                // and all iterators have been exhausted
                return result;
            }
            continue;
        } else {
            return result;
        }
    }
    // Should never reach here
    return result;
}