Examples with StorageDir alluxio.worker.block.meta.StorageDir used on opensource projects

Example 1 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class EvictorTestUtils method validCascadingPlan.

/**
   * Checks whether the plan of a cascading evictor is valid.
   *
   * A cascading evictor will try to free space by recursively moving blocks to next 1 tier and
   * evict blocks only in the bottom tier.
   *
   * The plan is invalid when the requested space can not be satisfied or lower level of tiers do
   * not have enough space to hold blocks moved from higher level of tiers.
   *
   * @param bytesToBeAvailable requested bytes to be available after eviction
   * @param plan the eviction plan, should not be empty
   * @param metaManager the metadata manager
   * @return true if the above requirements are satisfied, otherwise false
   * @throws BlockDoesNotExistException if a block for which metadata cannot be found is encountered
   */
// TODO(bin): Add a unit test for this method.
public static boolean validCascadingPlan(long bytesToBeAvailable, EvictionPlan plan, BlockMetadataManager metaManager) throws BlockDoesNotExistException {
    // reassure the plan is feasible: enough free space to satisfy bytesToBeAvailable, and enough
    // space in lower tier to move blocks in upper tier there
    // Map from dir to a pair of bytes to be available in this dir and bytes to move into this dir
    // after the plan taking action
    Map<StorageDir, Pair<Long, Long>> spaceInfoInDir = new HashMap<>();
    for (Pair<Long, BlockStoreLocation> blockInfo : plan.toEvict()) {
        BlockMeta block = metaManager.getBlockMeta(blockInfo.getFirst());
        StorageDir dir = block.getParentDir();
        if (spaceInfoInDir.containsKey(dir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(dir);
            spaceInfo.setFirst(spaceInfo.getFirst() + block.getBlockSize());
        } else {
            spaceInfoInDir.put(dir, new Pair<>(dir.getAvailableBytes() + block.getBlockSize(), 0L));
        }
    }
    for (BlockTransferInfo move : plan.toMove()) {
        long blockId = move.getBlockId();
        BlockMeta block = metaManager.getBlockMeta(blockId);
        long blockSize = block.getBlockSize();
        StorageDir srcDir = block.getParentDir();
        StorageDir destDir = metaManager.getDir(move.getDstLocation());
        if (spaceInfoInDir.containsKey(srcDir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(srcDir);
            spaceInfo.setFirst(spaceInfo.getFirst() + blockSize);
        } else {
            spaceInfoInDir.put(srcDir, new Pair<>(srcDir.getAvailableBytes() + blockSize, 0L));
        }
        if (spaceInfoInDir.containsKey(destDir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(destDir);
            spaceInfo.setSecond(spaceInfo.getSecond() + blockSize);
        } else {
            spaceInfoInDir.put(destDir, new Pair<>(destDir.getAvailableBytes(), blockSize));
        }
    }
    // the top tier among all tiers where blocks in the plan reside in
    int topTierOrdinal = Integer.MAX_VALUE;
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        topTierOrdinal = Math.min(topTierOrdinal, dir.getParentTier().getTierOrdinal());
    }
    // maximum bytes to be available in a dir in the top tier
    long maxSpace = Long.MIN_VALUE;
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        if (dir.getParentTier().getTierOrdinal() == topTierOrdinal) {
            Pair<Long, Long> space = spaceInfoInDir.get(dir);
            maxSpace = Math.max(maxSpace, space.getFirst() - space.getSecond());
        }
    }
    if (maxSpace < bytesToBeAvailable) {
        // plan is invalid because requested space can not be satisfied in the top tier
        return false;
    }
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        Pair<Long, Long> spaceInfo = spaceInfoInDir.get(dir);
        if (spaceInfo.getFirst() < spaceInfo.getSecond()) {
            // to be moved into this dir
            return false;
        }
    }
    return true;
}

Example 2 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class LRFUEvictorTest method cache.

private void cache(long sessionId, long blockId, long bytes, int tierLevel, int dirIdx) throws Exception {
    StorageDir dir = mMetaManager.getTiers().get(tierLevel).getDir(dirIdx);
    TieredBlockStoreTestUtils.cache(sessionId, blockId, bytes, dir, mMetaManager, mEvictor);
}

Example 3 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class BlockMetadataManagerViewTest method sameTierView.

/**
   * Tests that {@code BlockMetadataManagerView.getTierView(tierAlias)} returns the same
   * TierView as {@code new StorageTierView(mMetadataManager.getTier(tierAlias), this)}.
   */
@Test
public void sameTierView() {
    String tierAlias = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getTierAlias();
    StorageTierView tierView1 = mMetaManagerView.getTierView(tierAlias);
    // Do some operations on metadata
    StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getDir(TEST_DIR);
    BlockMeta blockMeta = new BlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir);
    try {
        dir.addBlockMeta(blockMeta);
    } catch (Exception e) {
        e.printStackTrace();
    }
    StorageTierView tierView2 = new StorageTierView(mMetaManager.getTier(tierAlias), mMetaManagerView);
    assertSameTierView(tierView1, tierView2);
}

Example 4 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class EvictorContractTest method needToEvict.

/**
   * Tests that an eviction plan is created when a directory is filled and the request size is the
   * capacity of the directory.
   */
@Test
public void needToEvict() throws Exception {
    // fill in a dir and request the capacity of the dir, all cached data in the dir should be
    // evicted.
    StorageDir dir = mTestDir;
    long capacityBytes = dir.getCapacityBytes();
    TieredBlockStoreTestUtils.cache(SESSION_ID, BLOCK_ID, capacityBytes, dir, mMetaManager, mEvictor);
    EvictionPlan plan = mEvictor.freeSpaceWithView(capacityBytes, dir.toBlockStoreLocation(), mManagerView);
    EvictorTestUtils.assertEvictionPlanValid(capacityBytes, plan, mMetaManager);
}

Example 5 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class EvictorContractTest method needToEvictAnyTier.

/**
   * Tests that an eviction plan is created when all capacity is used in each directory in all tiers
   * and the request size is the minimum capacity of all directories.
   */
@Test
public void needToEvictAnyTier() throws Exception {
    // cache data with size of "(capacity - 1)" in each dir in all tiers, request size of minimum
    // "capacity" of all dirs from anyTier
    long minCapacity = Long.MAX_VALUE;
    long blockId = BLOCK_ID;
    for (StorageTier tier : mMetaManager.getTiers()) {
        for (StorageDir dir : tier.getStorageDirs()) {
            long capacity = dir.getCapacityBytes();
            minCapacity = Math.min(minCapacity, capacity);
            TieredBlockStoreTestUtils.cache(SESSION_ID, blockId, capacity - 1, dir, mMetaManager, mEvictor);
            blockId++;
        }
    }
    EvictionPlan plan = mEvictor.freeSpaceWithView(minCapacity, BlockStoreLocation.anyTier(), mManagerView);
    EvictorTestUtils.assertEvictionPlanValid(minCapacity, plan, mMetaManager);
}