Examples with LocationBlockIdListEntry alluxio.grpc.LocationBlockIdListEntry used on opensource projects

Example 16 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMapIterator method next.

@Override
public List<LocationBlockIdListEntry> next() {
    List<LocationBlockIdListEntry> result = new ArrayList<>();
    int currentCounter = mCounter;
    int targetCounter = currentCounter + mBatchSize;
    while (mCounter < targetCounter) {
        // Find the BlockStoreLocation
        BlockStoreLocationProto currentLoc = mBlockStoreLocationProtoList.get(mCurrentBlockLocationIndex);
        Iterator<Long> currentIterator = mBlockLocationIteratorMap.get(currentLoc);
        // Generate the next batch
        // This method does NOT progress the pointers
        int spaceLeft = targetCounter - mCounter;
        LocationBlockIdListEntry batchFromThisTier = nextBatchFromTier(currentLoc, currentIterator, spaceLeft);
        result.add(batchFromThisTier);
        // Progress the iterator based on the break condition
        if (!currentIterator.hasNext()) {
            // We keep filling in using the next tier
            mCurrentBlockLocationIndex++;
            if (mCurrentBlockLocationIndex >= mBlockStoreLocationProtoList.size()) {
                // and all iterators have been exhausted
                return result;
            }
            continue;
        } else {
            return result;
        }
    }
    // Should never reach here
    return result;
}

Example 17 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockWorkerRegisterStreamIntegrationTest method requestsForWorker.

@Test
public void requestsForWorker() throws Exception {
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(WORKER_ID);
    // Verify the size and content of the requests
    int expectedBatchCount = (int) Math.ceil((TIER_BLOCK_TOTAL) / (double) BATCH_SIZE);
    Set<Long> containedBlockIds = new HashSet<>();
    assertEquals(expectedBatchCount, requestChunks.size());
    for (int i = 0; i < expectedBatchCount; i++) {
        RegisterWorkerPRequest request = requestChunks.get(i);
        assertEquals(WORKER_ID, request.getWorkerId());
        List<LocationBlockIdListEntry> entries = request.getCurrentBlocksList();
        int totalSize = 0;
        for (LocationBlockIdListEntry entry : entries) {
            totalSize += entry.getValue().getBlockIdCount();
            containedBlockIds.addAll(entry.getValue().getBlockIdList());
        }
        if (i != expectedBatchCount - 1) {
            assertEquals(BATCH_SIZE, totalSize);
        }
        // The 1st request contains metadata but the following do not
        if (i == 0) {
            assertEquals(USAGE_MAP, request.getUsedBytesOnTiersMap());
            assertEquals(CAPACITY_MAP, request.getTotalBytesOnTiersMap());
            Map<String, StorageList> lostMap = request.getLostStorageMap();
            assertEquals(1, lostMap.size());
            assertEquals(StorageList.newBuilder().build(), lostMap.get("MEM"));
            assertEquals(ImmutableList.of("MEM", "SSD", "HDD"), request.getStorageTiersList());
        } else {
            assertEquals(0, request.getStorageTiersCount());
            assertEquals(ImmutableMap.of(), request.getUsedBytesOnTiersMap());
            assertEquals(ImmutableMap.of(), request.getTotalBytesOnTiersMap());
            Map<String, StorageList> lostMap = request.getLostStorageMap();
            assertEquals(0, lostMap.size());
        }
    }
    assertEquals(containedBlockIds.size(), TIER_BLOCK_TOTAL);
}

Example 18 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMapIteratorTest method convertStream.

@Test
public void convertStream() {
    List<Integer> memList = ImmutableList.of(1000, 1000);
    List<Integer> ssdList = ImmutableList.of(10000, 10000);
    List<Integer> hddList = ImmutableList.of(20000, 20000);
    Map<String, List<Integer>> blockConfig = ImmutableMap.of("MEM", memList, "SSD", ssdList, "HDD", hddList);
    Map<BlockStoreLocation, List<Long>> blockMap = generateBlockIdOnTiers(blockConfig);
    BlockMapIterator iter = new BlockMapIterator(blockMap);
    while (iter.hasNext()) {
        List<LocationBlockIdListEntry> entries = iter.next();
        // Each batch should not contain duplicate locations
        Set<Block.BlockLocation> locations = new HashSet<>();
        for (LocationBlockIdListEntry e : entries) {
            // No duplicate
            Block.BlockLocation loc = Block.BlockLocation.newBuilder().setTier(e.getKey().getTierAlias()).setMediumType(e.getKey().getMediumType()).build();
            assertFalse(locations.contains(loc));
            locations.add(loc);
        }
    }
}

Example 19 with LocationBlockIdListEntry

use of alluxio.grpc.LocationBlockIdListEntry in project alluxio by Alluxio.

the class BlockMapIteratorTest method convertStreamMergedTiers.

@Test
public void convertStreamMergedTiers() {
    List<Integer> memList = ImmutableList.of(100, 100);
    List<Integer> ssdList = ImmutableList.of(101, 22);
    List<Integer> hddList = ImmutableList.of(10, 20);
    Map<String, List<Integer>> blockConfig = ImmutableMap.of("MEM", memList, "SSD", ssdList, "HDD", hddList);
    Map<BlockStoreLocation, List<Long>> blockMap = generateBlockIdOnTiers(blockConfig);
    BlockMapIterator iter = new BlockMapIterator(blockMap);
    while (iter.hasNext()) {
        List<LocationBlockIdListEntry> entries = iter.next();
        // Each batch should not contain duplicate locations
        Set<Block.BlockLocation> locations = new HashSet<>();
        for (LocationBlockIdListEntry e : entries) {
            // No duplicate
            Block.BlockLocation loc = Block.BlockLocation.newBuilder().setTier(e.getKey().getTierAlias()).setMediumType(e.getKey().getMediumType()).build();
            assertFalse(locations.contains(loc));
            locations.add(loc);
        }
    }
}