Examples with BlockStoreLocation alluxio.worker.block.BlockStoreLocation used on opensource projects

Example 6 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class LRUEvictorTest method cascadingEvictionTest3.

/**
   * Tests the behavior of moving blocks with the cascading eviction.
   */
@Test
public void cascadingEvictionTest3() throws Exception {
    // First Tier 2000, 3000
    // Second Tier 10000, 20000, 30000
    int blockSize = 1000;
    cache(SESSION_ID, 101, blockSize, 0, 0);
    cache(SESSION_ID, 102, blockSize, 0, 0);
    cache(SESSION_ID, 103, blockSize, 0, 1);
    cache(SESSION_ID, 104, blockSize, 0, 1);
    cache(SESSION_ID, 105, blockSize, 0, 1);
    cache(SESSION_ID, 106, 9500, 1, 2);
    // After caching blocks, the free space looks like
    // First Tier 0, 0
    // Second Tier 10000, 20000, 200500
    BlockStoreLocation anyDirInFirstTier = BlockStoreLocation.anyDirInTier("MEM");
    BlockStoreLocation firstDirSecondTier = new BlockStoreLocation("SSD", 0);
    BlockStoreLocation secondDirSecondTier = new BlockStoreLocation("SSD", 1);
    BlockStoreLocation thirdDirSecondTier = new BlockStoreLocation("SSD", 2);
    EvictionPlan plan = mEvictor.freeSpaceWithView(blockSize * 2, anyDirInFirstTier, mManagerView);
    Assert.assertNotNull(plan);
    Assert.assertEquals(0, plan.toEvict().size());
    Assert.assertEquals(2, plan.toMove().size());
    // 2 blocks to move. The first one should be moved the 3rd dir as it has max free space.
    long blockId = plan.toMove().get(0).getBlockId();
    Assert.assertEquals(101, blockId);
    BlockStoreLocation dstLocation = plan.toMove().get(0).getDstLocation();
    Assert.assertEquals(thirdDirSecondTier, dstLocation);
    // The second one should be moved the 2nd dir because after the first move the second dir
    // has the max free space.
    blockId = plan.toMove().get(1).getBlockId();
    Assert.assertEquals(102, blockId);
    dstLocation = plan.toMove().get(1).getDstLocation();
    Assert.assertEquals(secondDirSecondTier, dstLocation);
    cache(SESSION_ID, 107, 10000, 1, 0);
    cache(SESSION_ID, 108, 20000, 1, 1);
    cache(SESSION_ID, 109, 19000, 1, 2);
    access(106);
    // After caching more blocks, the free space looks like
    // First Tier 0, 0
    // Second Tier 0, 0, 1500
    plan = mEvictor.freeSpaceWithView(blockSize * 3, anyDirInFirstTier, mManagerView);
    Assert.assertNotNull(plan);
    Assert.assertEquals(1, plan.toEvict().size());
    Assert.assertEquals(3, plan.toMove().size());
    blockId = plan.toEvict().get(0).getFirst();
    Assert.assertEquals(107, blockId);
    // 3 blocks to move. The first one should be moved the 3rd dir as it has max free space.
    blockId = plan.toMove().get(0).getBlockId();
    Assert.assertEquals(103, blockId);
    dstLocation = plan.toMove().get(0).getDstLocation();
    Assert.assertEquals(thirdDirSecondTier, dstLocation);
    // The other two should be moved the 1st dir because the 1st dir has the max free space
    // after evicting block 107.
    blockId = plan.toMove().get(1).getBlockId();
    Assert.assertEquals(104, blockId);
    dstLocation = plan.toMove().get(1).getDstLocation();
    Assert.assertEquals(firstDirSecondTier, dstLocation);
    blockId = plan.toMove().get(2).getBlockId();
    Assert.assertEquals(105, blockId);
    dstLocation = plan.toMove().get(1).getDstLocation();
    Assert.assertEquals(firstDirSecondTier, dstLocation);
}

Example 7 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class PartialLRUEvictorTest method evictInBottomTier.

/**
   * Tests that the eviction in the bottom tier works.
   */
@Test
public void evictInBottomTier() throws Exception {
    int bottomTierLevel = TieredBlockStoreTestUtils.TIER_ORDINAL[TieredBlockStoreTestUtils.TIER_ORDINAL.length - 1];
    // capacity increases with index
    long[] bottomTierDirCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[bottomTierLevel];
    long smallestCapacity = bottomTierDirCapacity[0];
    long delta = smallestCapacity / 10;
    int nDir = bottomTierDirCapacity.length;
    // free space of StorageDir increases with Dir index
    for (int i = 0; i < nDir; i++) {
        cache(SESSION_ID, BLOCK_ID + i, bottomTierDirCapacity[i] - i * delta, bottomTierLevel, i);
    }
    BlockStoreLocation anyDirInBottomTier = BlockStoreLocation.anyDirInTier(TieredBlockStoreTestUtils.TIER_ALIAS[bottomTierLevel]);
    // free the StorageDir with max free space
    EvictionPlan plan = mEvictor.freeSpaceWithView(smallestCapacity, anyDirInBottomTier, mManagerView);
    Assert.assertNotNull(plan);
    Assert.assertTrue(plan.toMove().isEmpty());
    Assert.assertEquals(1, plan.toEvict().size());
    long toEvictBlockId = plan.toEvict().get(0).getFirst();
    Assert.assertEquals(BLOCK_ID + nDir - 1, toEvictBlockId);
}

Example 8 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class PartialLRUEvictorTest method cascadingEvictionTest1.

/**
   * Tests the cascading eviction with the first tier filled and the second tier empty resulting in
   * no eviction.
   */
@Test
public void cascadingEvictionTest1() throws Exception {
    // Two tiers, each dir in the second tier has more space than any dir in the first tier. Fill in
    // the first tier, leave the second tier empty. Request space from the first tier, blocks should
    // be moved from the first to the second tier without eviction.
    int firstTierOrdinal = TieredBlockStoreTestUtils.TIER_ORDINAL[0];
    long[] firstTierDirCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0];
    long smallestCapacity = firstTierDirCapacity[0];
    long delta = smallestCapacity / 10;
    int nDir = firstTierDirCapacity.length;
    for (int i = 0; i < nDir; i++) {
        cache(SESSION_ID, BLOCK_ID + i, firstTierDirCapacity[i] - delta * i, firstTierOrdinal, i);
    }
    BlockStoreLocation anyDirInFirstTier = BlockStoreLocation.anyDirInTier(TieredBlockStoreTestUtils.TIER_ALIAS[firstTierOrdinal]);
    EvictionPlan plan = mEvictor.freeSpaceWithView(smallestCapacity, anyDirInFirstTier, mManagerView);
    Assert.assertTrue(EvictorTestUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager));
    Assert.assertEquals(0, plan.toEvict().size());
    Assert.assertEquals(1, plan.toMove().size());
    long blockId = plan.toMove().get(0).getBlockId();
    Assert.assertEquals(BLOCK_ID + nDir - 1, blockId);
}

Example 9 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class StorageDirTest method toBlockStoreLocation.

/**
   * Tests the {@link StorageDir#toBlockStoreLocation()} method.
   */
@Test
public void toBlockStoreLocation() {
    StorageTier tier = mDir.getParentTier();
    Assert.assertEquals(new BlockStoreLocation(tier.getTierAlias(), mDir.getDirIndex()), mDir.toBlockStoreLocation());
}

Example 10 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class GreedyEvictor method freeSpaceWithView.

@Override
public EvictionPlan freeSpaceWithView(long availableBytes, BlockStoreLocation location, BlockMetadataManagerView view) {
    Preconditions.checkNotNull(location);
    Preconditions.checkNotNull(view);
    // 1. Select a StorageDirView that has enough capacity for required bytes.
    StorageDirView selectedDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        selectedDirView = selectEvictableDirFromAnyTier(view, availableBytes);
    } else {
        String tierAlias = location.tierAlias();
        StorageTierView tierView = view.getTierView(tierAlias);
        if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
            selectedDirView = selectEvictableDirFromTier(tierView, availableBytes);
        } else {
            int dirIndex = location.dir();
            StorageDirView dir = tierView.getDirView(dirIndex);
            if (canEvictBlocksFromDir(dir, availableBytes)) {
                selectedDirView = dir;
            }
        }
    }
    if (selectedDirView == null) {
        LOG.error("Failed to freeSpace: No StorageDirView has enough capacity of {} bytes", availableBytes);
        return null;
    }
    // 2. Check if the selected StorageDirView already has enough space.
    List<BlockTransferInfo> toTransfer = new ArrayList<>();
    List<Pair<Long, BlockStoreLocation>> toEvict = new ArrayList<>();
    long bytesAvailableInDir = selectedDirView.getAvailableBytes();
    if (bytesAvailableInDir >= availableBytes) {
        // No need to evict anything, return an eviction plan with empty instructions.
        return new EvictionPlan(toTransfer, toEvict);
    }
    // 3. Collect victim blocks from the selected StorageDirView. They could either be evicted or
    // moved.
    List<BlockMeta> victimBlocks = new ArrayList<>();
    for (BlockMeta block : selectedDirView.getEvictableBlocks()) {
        victimBlocks.add(block);
        bytesAvailableInDir += block.getBlockSize();
        if (bytesAvailableInDir >= availableBytes) {
            break;
        }
    }
    // 4. Make best effort to transfer victim blocks to lower tiers rather than evict them.
    Map<StorageDirView, Long> pendingBytesInDir = new HashMap<>();
    for (BlockMeta block : victimBlocks) {
        // TODO(qifan): Should avoid calling getParentDir.
        String fromTierAlias = block.getParentDir().getParentTier().getTierAlias();
        List<StorageTierView> candidateTiers = view.getTierViewsBelow(fromTierAlias);
        StorageDirView dstDir = selectAvailableDir(block, candidateTiers, pendingBytesInDir);
        if (dstDir == null) {
            // Not possible to transfer
            toEvict.add(new Pair<>(block.getBlockId(), block.getBlockLocation()));
        } else {
            StorageTierView dstTier = dstDir.getParentTierView();
            toTransfer.add(new BlockTransferInfo(block.getBlockId(), block.getBlockLocation(), new BlockStoreLocation(dstTier.getTierViewAlias(), dstDir.getDirViewIndex())));
            if (pendingBytesInDir.containsKey(dstDir)) {
                pendingBytesInDir.put(dstDir, pendingBytesInDir.get(dstDir) + block.getBlockSize());
            } else {
                pendingBytesInDir.put(dstDir, block.getBlockSize());
            }
        }
    }
    return new EvictionPlan(toTransfer, toEvict);
}