Examples with BlockStoreLocation alluxio.worker.block.BlockStoreLocation used on opensource projects

Example 11 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class AbstractBlockMetaTest method getBlockLocation.

/**
   * Tests the {@link AbstractBlockMeta#getBlockLocation()} method.
   */
@Test
public void getBlockLocation() {
    BlockStoreLocation expectedLocation = new BlockStoreLocation(mTier.getTierAlias(), mDir.getDirIndex());
    Assert.assertEquals(expectedLocation, mBlockMeta.getBlockLocation());
}

Example 12 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class EvictorContractTest method requestSpaceLargerThanCapacity.

/**
   * Tests that no eviction plan is available when requesting more space than capacity available.
   */
@Test
public void requestSpaceLargerThanCapacity() throws Exception {
    // cache data in a dir
    long totalCapacity = mMetaManager.getAvailableBytes(BlockStoreLocation.anyTier());
    StorageDir dir = mTestDir;
    BlockStoreLocation dirLocation = dir.toBlockStoreLocation();
    long dirCapacity = mMetaManager.getAvailableBytes(dirLocation);
    TieredBlockStoreTestUtils.cache(SESSION_ID, BLOCK_ID, dirCapacity, dir, mMetaManager, mEvictor);
    // request space larger than total capacity, no eviction plan should be available
    Assert.assertNull(mEvictor.freeSpaceWithView(totalCapacity + 1, BlockStoreLocation.anyTier(), mManagerView));
    // request space larger than capacity for the random directory, no eviction plan should be
    // available
    Assert.assertNull(mEvictor.freeSpaceWithView(dirCapacity + 1, dirLocation, mManagerView));
}

Example 13 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class EvictorTestUtils method requestSpaceSatisfied.

/**
   * Whether the plan can satisfy the requested free bytes to be available, assume all blocks in the
   * plan are in the same dir.
   *
   * @param bytesToBeAvailable the requested bytes to be available
   * @param plan the eviction plan, should not be null
   * @param meta the metadata manager
   * @return true if the request can be satisfied otherwise false
   * @throws alluxio.exception.BlockDoesNotExistException if can not get metadata of a block
   */
public static boolean requestSpaceSatisfied(long bytesToBeAvailable, EvictionPlan plan, BlockMetadataManager meta) throws BlockDoesNotExistException {
    Preconditions.checkNotNull(plan);
    List<Long> blockIds = new ArrayList<>();
    for (Pair<Long, BlockStoreLocation> evict : plan.toEvict()) {
        blockIds.add(evict.getFirst());
    }
    for (BlockTransferInfo move : plan.toMove()) {
        blockIds.add(move.getBlockId());
    }
    long evictedOrMovedBytes = 0;
    for (long blockId : blockIds) {
        evictedOrMovedBytes += meta.getBlockMeta(blockId).getBlockSize();
    }
    BlockStoreLocation location = meta.getBlockMeta(blockIds.get(0)).getParentDir().toBlockStoreLocation();
    return (meta.getAvailableBytes(location) + evictedOrMovedBytes) >= bytesToBeAvailable;
}

Example 14 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class EvictorTestUtils method blocksInTheSameDir.

/**
   * Whether blocks in the {@link EvictionPlan} are in the same {@link StorageDir}.
   *
   * @param plan the eviction plan
   * @param meta the metadata manager
   * @return true if blocks are in the same dir otherwise false
   * @throws BlockDoesNotExistException if fail to get metadata of a block
   */
public static boolean blocksInTheSameDir(EvictionPlan plan, BlockMetadataManager meta) throws BlockDoesNotExistException {
    Preconditions.checkNotNull(plan);
    StorageDir dir = null;
    List<Long> blockIds = new ArrayList<>();
    for (Pair<Long, BlockStoreLocation> evict : plan.toEvict()) {
        blockIds.add(evict.getFirst());
    }
    for (BlockTransferInfo move : plan.toMove()) {
        blockIds.add(move.getBlockId());
    }
    for (long blockId : blockIds) {
        StorageDir blockDir = meta.getBlockMeta(blockId).getParentDir();
        if (dir == null) {
            dir = blockDir;
        } else if (dir != blockDir) {
            return false;
        }
    }
    return true;
}

Example 15 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class LRFUEvictorTest method cascadingEvictionTest2.

/**
   * Tests the cascading eviction with the first and second tier filled resulting in blocks in the
   * second tier are evicted.
   */
@Test
public void cascadingEvictionTest2() throws Exception {
    // Two tiers, the second tier has more dirs than the first tier and each dir in the second tier
    // has more space than any dir in the first tier. Fill in all dirs and request space from the
    // first tier, blocks should be moved from the first to the second tier, and some blocks in the
    // second tier should be evicted to hold blocks moved from the first tier.
    long blockId = BLOCK_ID;
    long totalBlocks = 0;
    for (int tierOrdinal : TieredBlockStoreTestUtils.TIER_ORDINAL) {
        totalBlocks += TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[tierOrdinal].length;
    }
    Map<Long, Double> blockIdToCRF = new HashMap<>();
    for (int tierOrdinal : TieredBlockStoreTestUtils.TIER_ORDINAL) {
        long[] tierCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[tierOrdinal];
        for (int dirIdx = 0; dirIdx < tierCapacity.length; dirIdx++) {
            cache(SESSION_ID, blockId, tierCapacity[dirIdx], tierOrdinal, dirIdx);
            // update CRF of blocks when blocks are committed
            blockIdToCRF.put(blockId, calculateAccessWeight(totalBlocks - 1 - (blockId - BLOCK_ID)));
            blockId++;
        }
    }
    // Update CRF of all blocks during each access
    for (int i = 0; i < totalBlocks; i++) {
        for (int j = 0; j <= i; j++) {
            access(BLOCK_ID + j);
            for (int k = 0; k < totalBlocks; k++) {
                if (k == j) {
                    blockIdToCRF.put(BLOCK_ID + k, blockIdToCRF.get(BLOCK_ID + k) * calculateAccessWeight(1L) + 1.0);
                } else {
                    blockIdToCRF.put(BLOCK_ID + k, blockIdToCRF.get(BLOCK_ID + k) * calculateAccessWeight(1L));
                }
            }
        }
    }
    List<Map.Entry<Long, Double>> blockCRF = getSortedCRF(blockIdToCRF);
    // sorted blocks in the first tier
    List<Long> blocksInFirstTier = new ArrayList<>();
    // sorted blocks in the second tier
    List<Long> blocksInSecondTier = new ArrayList<>();
    for (int i = 0; i < blockCRF.size(); i++) {
        long block = blockCRF.get(i).getKey();
        if (block - BLOCK_ID < TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0].length) {
            blocksInFirstTier.add(block);
        } else if (block - BLOCK_ID < TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0].length + TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[1].length) {
            blocksInSecondTier.add(block);
        }
    }
    BlockStoreLocation anyDirInFirstTier = BlockStoreLocation.anyDirInTier(TieredBlockStoreTestUtils.TIER_ALIAS[0]);
    int nDirInFirstTier = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0].length;
    long smallestCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0][0];
    for (int i = 0; i < nDirInFirstTier; i++) {
        EvictionPlan plan = mEvictor.freeSpaceWithView(smallestCapacity, anyDirInFirstTier, mManagerView);
        Assert.assertTrue(EvictorTestUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager));
        // block with minimum CRF in the first tier needs to be moved to the second tier
        Assert.assertEquals(1, plan.toMove().size());
        long blockIdMovedInFirstTier = plan.toMove().get(0).getBlockId();
        long objectBlockIdInFirstTier = blocksInFirstTier.get(i);
        Assert.assertEquals(objectBlockIdInFirstTier, blockIdMovedInFirstTier);
        // cached block with minimum CRF in the second tier will be evicted to hold blocks moved
        // from first tier
        Assert.assertEquals(1, plan.toEvict().size());
        long blockIdEvictedInSecondTier = plan.toEvict().get(0).getFirst();
        long objectBlockIdInSecondTier = blocksInSecondTier.get(i);
        Assert.assertEquals(objectBlockIdInSecondTier, blockIdEvictedInSecondTier);
        // update CRF of the chosen blocks in case that they are chosen again
        for (int j = 0; j < totalBlocks; j++) {
            access(blockIdMovedInFirstTier);
        }
        for (int j = 0; j < totalBlocks; j++) {
            access(blockIdEvictedInSecondTier);
        }
    }
}