Examples with BlockMetadataEvictorView alluxio.worker.block.BlockMetadataEvictorView used on opensource projects

Example 1 with BlockMetadataEvictorView

use of alluxio.worker.block.BlockMetadataEvictorView in project alluxio by Alluxio.

the class StorageDirViewTest method before.

/**
 * Sets up all dependencies before a test runs.
 */
@Before
public void before() throws Exception {
    File tempFolder = mTestFolder.newFolder();
    BlockMetadataManager metaManager = TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath());
    mMetadataView = Mockito.spy(new BlockMetadataEvictorView(metaManager, new HashSet<Long>(), new HashSet<Long>()));
    StorageTier testTier = metaManager.getTiers().get(TEST_TIER_LEVEL);
    mTestDir = testTier.getDir(TEST_DIR);
    mTestTierView = new StorageTierEvictorView(testTier, mMetadataView);
    mTestDirView = new StorageDirEvictorView(mTestDir, mTestTierView, mMetadataView);
}

Example 2 with BlockMetadataEvictorView

use of alluxio.worker.block.BlockMetadataEvictorView in project alluxio by Alluxio.

the class StorageTierViewTest method before.

/**
 * Sets up all dependencies before a test runs.
 */
@Before
public void before() throws Exception {
    File tempFolder = mTestFolder.newFolder();
    BlockMetadataManager metaManager = TieredBlockStoreTestUtils.defaultMetadataManager(tempFolder.getAbsolutePath());
    mMetadataView = new BlockMetadataEvictorView(metaManager, new HashSet<Long>(), new HashSet<Long>());
    mTestTier = metaManager.getTiers().get(TEST_TIER_LEVEL);
    mTestTierView = new StorageTierEvictorView(mTestTier, mMetadataView);
}

Example 3 with BlockMetadataEvictorView

use of alluxio.worker.block.BlockMetadataEvictorView in project alluxio by Alluxio.

the class EmulatingBlockIterator method getIterator.

@Override
public Iterator<Long> getIterator(BlockStoreLocation location, BlockOrder order) {
    /**
     * Invoke the evictor for the location with configured free space threshold.
     * From the generated plan, extract the order imposed by cascading eviction logic.
     *
     * This emulation assumes that toEvict() and toMove() lists of the plan
     * is populated based on an internal order maintained by the evictor implementation.
     */
    // Generate the plan as every block being evictable.
    EvictionPlan evictionPlan = mEvictor.freeSpaceWithView(mReservedSpaces.get(location.tierAlias()), location, new BlockMetadataEvictorView(mMetadataManager, Collections.emptySet(), Collections.emptySet()), Evictor.Mode.BEST_EFFORT);
    // Extract evicted blocks in order.
    List<Long> toEvict = Collections.emptyList();
    if (evictionPlan.toEvict() != null) {
        toEvict = evictionPlan.toEvict().stream().map((kvp) -> kvp.getFirst()).collect(Collectors.toList());
    }
    // Extract moved blocks in order.
    List<Long> toMove = Collections.emptyList();
    if (evictionPlan.toMove() != null) {
        toMove = evictionPlan.toMove().stream().map((kvp) -> kvp.getSrcBlockId()).collect(Collectors.toList());
    }
    // Select which list to feed as iterator.
    // For the lowest tier blocks to evict will be considered.
    // For the other cases blocks to move will be considered.
    List<Long> iteratorList = toEvict;
    if (!toMove.isEmpty()) {
        iteratorList = toMove;
    }
    // Reverse the list if requested.
    if (order == BlockOrder.REVERSE) {
        Collections.reverse(iteratorList);
    }
    // Return an iterator from combined block id list.
    return iteratorList.iterator();
}

Example 4 with BlockMetadataEvictorView

use of alluxio.worker.block.BlockMetadataEvictorView in project alluxio by Alluxio.

the class TierManagementTaskProvider method findNextTask.

/**
 * @return the next tier management task that is required
 */
private TierManagementTaskType findNextTask() {
    // Fetch the configuration for supported tasks types.
    boolean alignEnabled = ServerConfiguration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED);
    boolean swapRestoreEnabled = ServerConfiguration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_SWAP_RESTORE_ENABLED);
    boolean promotionEnabled = ServerConfiguration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED);
    // Return swap-restore task if marked.
    if (swapRestoreEnabled && sSwapRestoreRequired) {
        setSwapRestoreRequired(false);
        LOG.debug("Swap-restore needed.");
        return TierManagementTaskType.SWAP_RESTORE;
    }
    // Acquire a recent evictor view.
    BlockMetadataEvictorView evictorView = mEvictorViewSupplier.get();
    // Iterate all tier intersections and decide which task to run.
    for (Pair<BlockStoreLocation, BlockStoreLocation> intersection : mMetadataManager.getStorageTierAssoc().intersectionList()) {
        // Check if the intersection needs alignment.
        if (alignEnabled && !mMetadataManager.getBlockIterator().aligned(intersection.getFirst(), intersection.getSecond(), BlockOrder.NATURAL, (blockId) -> !evictorView.isBlockEvictable(blockId))) {
            LOG.debug("Alignment needed between: {} - {}", intersection.getFirst().tierAlias(), intersection.getSecond().tierAlias());
            return TierManagementTaskType.ALIGN;
        }
        // Check if the intersection allows for promotions.
        if (promotionEnabled) {
            StorageTier highTier = mMetadataManager.getTier(intersection.getFirst().tierAlias());
            // Current used percent on the high tier of intersection.
            double currentUsedRatio = 1.0 - (double) highTier.getAvailableBytes() / highTier.getCapacityBytes();
            // Configured promotion quota percent for tiers.
            double quotaRatio = (double) ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT) / 100;
            // Check if the high tier allows for promotions.
            if (currentUsedRatio < quotaRatio) {
                // Check if there is anything to move from lower tier.
                Iterator<Long> lowBlocks = mMetadataManager.getBlockIterator().getIterator(intersection.getSecond(), BlockOrder.REVERSE);
                while (lowBlocks.hasNext()) {
                    if (evictorView.isBlockEvictable(lowBlocks.next())) {
                        LOG.debug("Promotions needed from {} to {}", intersection.getSecond().tierAlias(), intersection.getFirst().tierAlias());
                        return TierManagementTaskType.PROMOTE;
                    }
                }
            }
        }
    }
    // No tier management task is required/allowed.
    return TierManagementTaskType.NONE;
}