Examples with BlockStoreLocation alluxio.worker.block.BlockStoreLocation used on opensource projects

Example 1 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class EvictorTestUtils method validCascadingPlan.

/**
   * Checks whether the plan of a cascading evictor is valid.
   *
   * A cascading evictor will try to free space by recursively moving blocks to next 1 tier and
   * evict blocks only in the bottom tier.
   *
   * The plan is invalid when the requested space can not be satisfied or lower level of tiers do
   * not have enough space to hold blocks moved from higher level of tiers.
   *
   * @param bytesToBeAvailable requested bytes to be available after eviction
   * @param plan the eviction plan, should not be empty
   * @param metaManager the metadata manager
   * @return true if the above requirements are satisfied, otherwise false
   * @throws BlockDoesNotExistException if a block for which metadata cannot be found is encountered
   */
// TODO(bin): Add a unit test for this method.
public static boolean validCascadingPlan(long bytesToBeAvailable, EvictionPlan plan, BlockMetadataManager metaManager) throws BlockDoesNotExistException {
    // reassure the plan is feasible: enough free space to satisfy bytesToBeAvailable, and enough
    // space in lower tier to move blocks in upper tier there
    // Map from dir to a pair of bytes to be available in this dir and bytes to move into this dir
    // after the plan taking action
    Map<StorageDir, Pair<Long, Long>> spaceInfoInDir = new HashMap<>();
    for (Pair<Long, BlockStoreLocation> blockInfo : plan.toEvict()) {
        BlockMeta block = metaManager.getBlockMeta(blockInfo.getFirst());
        StorageDir dir = block.getParentDir();
        if (spaceInfoInDir.containsKey(dir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(dir);
            spaceInfo.setFirst(spaceInfo.getFirst() + block.getBlockSize());
        } else {
            spaceInfoInDir.put(dir, new Pair<>(dir.getAvailableBytes() + block.getBlockSize(), 0L));
        }
    }
    for (BlockTransferInfo move : plan.toMove()) {
        long blockId = move.getBlockId();
        BlockMeta block = metaManager.getBlockMeta(blockId);
        long blockSize = block.getBlockSize();
        StorageDir srcDir = block.getParentDir();
        StorageDir destDir = metaManager.getDir(move.getDstLocation());
        if (spaceInfoInDir.containsKey(srcDir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(srcDir);
            spaceInfo.setFirst(spaceInfo.getFirst() + blockSize);
        } else {
            spaceInfoInDir.put(srcDir, new Pair<>(srcDir.getAvailableBytes() + blockSize, 0L));
        }
        if (spaceInfoInDir.containsKey(destDir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(destDir);
            spaceInfo.setSecond(spaceInfo.getSecond() + blockSize);
        } else {
            spaceInfoInDir.put(destDir, new Pair<>(destDir.getAvailableBytes(), blockSize));
        }
    }
    // the top tier among all tiers where blocks in the plan reside in
    int topTierOrdinal = Integer.MAX_VALUE;
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        topTierOrdinal = Math.min(topTierOrdinal, dir.getParentTier().getTierOrdinal());
    }
    // maximum bytes to be available in a dir in the top tier
    long maxSpace = Long.MIN_VALUE;
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        if (dir.getParentTier().getTierOrdinal() == topTierOrdinal) {
            Pair<Long, Long> space = spaceInfoInDir.get(dir);
            maxSpace = Math.max(maxSpace, space.getFirst() - space.getSecond());
        }
    }
    if (maxSpace < bytesToBeAvailable) {
        // plan is invalid because requested space can not be satisfied in the top tier
        return false;
    }
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        Pair<Long, Long> spaceInfo = spaceInfoInDir.get(dir);
        if (spaceInfo.getFirst() < spaceInfo.getSecond()) {
            // to be moved into this dir
            return false;
        }
    }
    return true;
}

Example 2 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class LRFUEvictorTest method evictInBottomTier.

/**
   * Tests that the eviction in the bottom tier works.
   */
@Test
public void evictInBottomTier() throws Exception {
    int bottomTierOrdinal = TieredBlockStoreTestUtils.TIER_ORDINAL[TieredBlockStoreTestUtils.TIER_ORDINAL.length - 1];
    Map<Long, Double> blockIdToCRF = new HashMap<>();
    // capacity increases with index
    long[] bottomTierDirCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[bottomTierOrdinal];
    int nDir = bottomTierDirCapacity.length;
    // fill in dirs from larger to smaller capacity with blockId equal to BLOCK_ID plus dir index
    for (int i = 0; i < nDir; i++) {
        cache(SESSION_ID, BLOCK_ID + i, bottomTierDirCapacity[i], bottomTierOrdinal, i);
        // update CRF of blocks when blocks are committed
        blockIdToCRF.put(BLOCK_ID + i, calculateAccessWeight(nDir - 1 - i));
    }
    // during each access
    for (int i = 0; i < nDir; i++) {
        for (int j = 0; j <= i; j++) {
            access(BLOCK_ID + j);
            for (int k = 0; k < nDir; k++) {
                if (k == j) {
                    blockIdToCRF.put(BLOCK_ID + k, blockIdToCRF.get(BLOCK_ID + k) * calculateAccessWeight(1L) + 1.0);
                } else {
                    blockIdToCRF.put(BLOCK_ID + k, blockIdToCRF.get(BLOCK_ID + k) * calculateAccessWeight(1L));
                }
            }
        }
    }
    // sort blocks in ascending order of CRF
    List<Entry<Long, Double>> blockCRF = getSortedCRF(blockIdToCRF);
    BlockStoreLocation anyDirInBottomTier = BlockStoreLocation.anyDirInTier(TieredBlockStoreTestUtils.TIER_ALIAS[bottomTierOrdinal]);
    // to evict blocks from should be in the same order as sorted blockCRF
    for (int i = 0; i < nDir; i++) {
        EvictionPlan plan = mEvictor.freeSpaceWithView(bottomTierDirCapacity[0], anyDirInBottomTier, mManagerView);
        Assert.assertNotNull(plan);
        Assert.assertTrue(plan.toMove().isEmpty());
        Assert.assertEquals(1, plan.toEvict().size());
        long toEvictBlockId = plan.toEvict().get(0).getFirst();
        long objectBlockId = blockCRF.get(i).getKey();
        Assert.assertEquals(objectBlockId + " " + toEvictBlockId, objectBlockId, toEvictBlockId);
        // update CRF of the chosen block in case that it is chosen again
        for (int j = 0; j < nDir; j++) {
            access(toEvictBlockId);
        }
    }
}

Example 3 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class LRFUEvictorTest method cascadingEvictionTest1.

/**
   * Tests the cascading eviction with the first tier filled and the second tier empty resulting in
   * no eviction.
   */
@Test
public void cascadingEvictionTest1() throws Exception {
    // Two tiers, each dir in the second tier has more space than any dir in the first tier. Fill in
    // the first tier, leave the second tier empty. Request space from the first tier, blocks should
    // be moved from the first to the second tier without eviction.
    int firstTierOrdinal = TieredBlockStoreTestUtils.TIER_ORDINAL[0];
    long[] firstTierDirCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0];
    int nDir = firstTierDirCapacity.length;
    Map<Long, Double> blockIdToCRF = new HashMap<>();
    for (int i = 0; i < nDir; i++) {
        cache(SESSION_ID, BLOCK_ID + i, firstTierDirCapacity[i], firstTierOrdinal, i);
        // update CRF of blocks when blocks are committed
        blockIdToCRF.put(BLOCK_ID + i, calculateAccessWeight(nDir - 1 - i));
    }
    // during each access
    for (int i = 0; i < nDir; i++) {
        for (int j = 0; j <= i; j++) {
            access(BLOCK_ID + j);
            for (int k = 0; k < nDir; k++) {
                if (k == j) {
                    blockIdToCRF.put(BLOCK_ID + k, blockIdToCRF.get(BLOCK_ID + k) * calculateAccessWeight(1L) + 1.0);
                } else {
                    blockIdToCRF.put(BLOCK_ID + k, blockIdToCRF.get(BLOCK_ID + k) * calculateAccessWeight(1L));
                }
            }
        }
    }
    List<Map.Entry<Long, Double>> blockCRF = getSortedCRF(blockIdToCRF);
    BlockStoreLocation anyDirInFirstTier = BlockStoreLocation.anyDirInTier(TieredBlockStoreTestUtils.TIER_ALIAS[firstTierOrdinal]);
    long smallestCapacity = firstTierDirCapacity[0];
    // to move blocks from should be in the same order as sorted blockCRF
    for (int i = 0; i < nDir; i++) {
        EvictionPlan plan = mEvictor.freeSpaceWithView(smallestCapacity, anyDirInFirstTier, mManagerView);
        Assert.assertTrue(EvictorTestUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager));
        Assert.assertEquals(0, plan.toEvict().size());
        Assert.assertEquals(1, plan.toMove().size());
        long blockId = plan.toMove().get(0).getBlockId();
        long objectBlockId = blockCRF.get(i).getKey();
        Assert.assertEquals(objectBlockId, blockId);
        // update CRF of the chosen block in case that it is chosen again
        for (int j = 0; j < nDir; j++) {
            access(objectBlockId);
        }
    }
}

Example 4 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class LRUEvictorTest method evictInBottomTier.

/**
   * Tests that the eviction in the bottom tier works.
   */
@Test
public void evictInBottomTier() throws Exception {
    int bottomTierOrdinal = TieredBlockStoreTestUtils.TIER_ORDINAL[TieredBlockStoreTestUtils.TIER_ORDINAL.length - 1];
    // capacity increases with index
    long[] bottomTierDirCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[bottomTierOrdinal];
    int nDir = bottomTierDirCapacity.length;
    // fill in dirs from larger to smaller capacity with blockId equal to BLOCK_ID plus dir index
    for (int i = nDir - 1; i >= 0; i--) {
        cache(SESSION_ID, BLOCK_ID + i, bottomTierDirCapacity[i], bottomTierOrdinal, i);
    }
    BlockStoreLocation anyDirInBottomTier = BlockStoreLocation.anyDirInTier(TieredBlockStoreTestUtils.TIER_ALIAS[bottomTierOrdinal]);
    // to evict blocks from should be in the same order as caching
    for (int i = nDir - 1; i >= 0; i--) {
        EvictionPlan plan = mEvictor.freeSpaceWithView(bottomTierDirCapacity[0], anyDirInBottomTier, mManagerView);
        Assert.assertNotNull(plan);
        Assert.assertTrue(plan.toMove().isEmpty());
        Assert.assertEquals(1, plan.toEvict().size());
        long toEvictBlockId = plan.toEvict().get(0).getFirst();
        Assert.assertEquals(BLOCK_ID + i, toEvictBlockId);
        access(toEvictBlockId);
    }
}

Example 5 with BlockStoreLocation

use of alluxio.worker.block.BlockStoreLocation in project alluxio by Alluxio.

the class LRUEvictorTest method cascadingEvictionTest2.

/**
   * Tests the cascading eviction with the first and second tier filled resulting in blocks in the
   * second tier are evicted.
   */
@Test
public void cascadingEvictionTest2() throws Exception {
    // Two tiers, the second tier has more dirs than the first tier and each dir in the second tier
    // has more space than any dir in the first tier. Fill in all dirs and request space from the
    // first tier, blocks should be moved from the first to the second tier, and some blocks in the
    // second tier should be evicted to hold blocks moved from the first tier.
    long blockId = BLOCK_ID;
    for (int tierOrdinal : TieredBlockStoreTestUtils.TIER_ORDINAL) {
        long[] tierCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[tierOrdinal];
        for (int dirIdx = 0; dirIdx < tierCapacity.length; dirIdx++) {
            cache(SESSION_ID, blockId, tierCapacity[dirIdx], tierOrdinal, dirIdx);
            blockId++;
        }
    }
    BlockStoreLocation anyDirInFirstTier = BlockStoreLocation.anyDirInTier(TieredBlockStoreTestUtils.TIER_ALIAS[0]);
    int nDirInFirstTier = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0].length;
    long smallestCapacity = TieredBlockStoreTestUtils.TIER_CAPACITY_BYTES[0][0];
    for (int i = 0; i < nDirInFirstTier; i++) {
        EvictionPlan plan = mEvictor.freeSpaceWithView(smallestCapacity, anyDirInFirstTier, mManagerView);
        Assert.assertTrue(EvictorTestUtils.validCascadingPlan(smallestCapacity, plan, mMetaManager));
        // least recently used block in the first tier needs to be moved to the second tier
        Assert.assertEquals(1, plan.toMove().size());
        long blockIdMovedInFirstTier = plan.toMove().get(0).getBlockId();
        Assert.assertEquals(BLOCK_ID + i, blockIdMovedInFirstTier);
        // least recently used cached block in the second tier will be evicted to hold blocks moved
        // from first tier
        Assert.assertEquals(1, plan.toEvict().size());
        long blockIdEvictedInSecondTier = plan.toEvict().get(0).getFirst();
        Assert.assertEquals(BLOCK_ID + nDirInFirstTier + i, blockIdEvictedInSecondTier);
        access(blockIdMovedInFirstTier);
        access(blockIdEvictedInSecondTier);
    }
}