Examples with StorageTier alluxio.worker.block.meta.StorageTier used on opensource projects

Example 21 with StorageTier

use of alluxio.worker.block.meta.StorageTier in project alluxio by Alluxio.

the class PromoteTask method getTransferInfos.

/**
 * @return list of block transfers
 */
private List<BlockTransferInfo> getTransferInfos(Iterator<Long> iterator, BlockStoreLocation tierUpLocation, BlockStoreLocation tierDownLocation) {
    // Acquire promotion range from the configuration.
    // This will limit promotions in single task run.
    final int promoteRange = ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_RANGE);
    // Tier for where promotions are going to.
    StorageTier tierUp = mMetadataManager.getTier(tierUpLocation.tierAlias());
    // Get quota for promotions.
    int promotionQuota = ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT);
    Preconditions.checkArgument(promotionQuota >= 0 && promotionQuota <= 100, "Invalid promotion quota percent");
    double quotaRatio = (double) promotionQuota / 100;
    // List to store transfer infos for selected blocks.
    List<BlockTransferInfo> transferInfos = new LinkedList<>();
    // Projected allocation for selected blocks.
    long bytesToAllocate = 0;
    // Gather blocks from iterator upto configured free space limit.
    while (iterator.hasNext() && transferInfos.size() < promoteRange) {
        // Stop moving if reached promotion quota on higher tier.
        double projectedUsedRatio = 1.0 - ((double) (tierUp.getAvailableBytes() - bytesToAllocate) / tierUp.getCapacityBytes());
        if (projectedUsedRatio >= quotaRatio) {
            break;
        }
        long blockId = iterator.next();
        // Read block info and store it.
        try {
            BlockMeta blockMeta = mEvictorView.getBlockMeta(blockId);
            if (blockMeta == null) {
                LOG.debug("Block:{} exist but not available for promotion.", blockId);
                continue;
            }
            bytesToAllocate += blockMeta.getBlockSize();
            transferInfos.add(BlockTransferInfo.createMove(blockMeta.getBlockLocation(), blockId, tierUpLocation));
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Failed to find location of a block:{}. Error: {}", blockId, e);
            continue;
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated {} promotions from {} to {}.\n" + "Promotions transfers:\n ->{}", transferInfos.size(), tierDownLocation.tierAlias(), tierUpLocation.tierAlias(), transferInfos.stream().map(Objects::toString).collect(Collectors.joining("\n ->")));
    }
    return transferInfos;
}

Example 22 with StorageTier

use of alluxio.worker.block.meta.StorageTier in project alluxio by Alluxio.

the class TierManagementTaskProvider method findNextTask.

/**
 * @return the next tier management task that is required
 */
private TierManagementTaskType findNextTask() {
    // Fetch the configuration for supported tasks types.
    boolean alignEnabled = ServerConfiguration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_ALIGN_ENABLED);
    boolean swapRestoreEnabled = ServerConfiguration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_SWAP_RESTORE_ENABLED);
    boolean promotionEnabled = ServerConfiguration.getBoolean(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_ENABLED);
    // Return swap-restore task if marked.
    if (swapRestoreEnabled && sSwapRestoreRequired) {
        setSwapRestoreRequired(false);
        LOG.debug("Swap-restore needed.");
        return TierManagementTaskType.SWAP_RESTORE;
    }
    // Acquire a recent evictor view.
    BlockMetadataEvictorView evictorView = mEvictorViewSupplier.get();
    // Iterate all tier intersections and decide which task to run.
    for (Pair<BlockStoreLocation, BlockStoreLocation> intersection : mMetadataManager.getStorageTierAssoc().intersectionList()) {
        // Check if the intersection needs alignment.
        if (alignEnabled && !mMetadataManager.getBlockIterator().aligned(intersection.getFirst(), intersection.getSecond(), BlockOrder.NATURAL, (blockId) -> !evictorView.isBlockEvictable(blockId))) {
            LOG.debug("Alignment needed between: {} - {}", intersection.getFirst().tierAlias(), intersection.getSecond().tierAlias());
            return TierManagementTaskType.ALIGN;
        }
        // Check if the intersection allows for promotions.
        if (promotionEnabled) {
            StorageTier highTier = mMetadataManager.getTier(intersection.getFirst().tierAlias());
            // Current used percent on the high tier of intersection.
            double currentUsedRatio = 1.0 - (double) highTier.getAvailableBytes() / highTier.getCapacityBytes();
            // Configured promotion quota percent for tiers.
            double quotaRatio = (double) ServerConfiguration.getInt(PropertyKey.WORKER_MANAGEMENT_TIER_PROMOTE_QUOTA_PERCENT) / 100;
            // Check if the high tier allows for promotions.
            if (currentUsedRatio < quotaRatio) {
                // Check if there is anything to move from lower tier.
                Iterator<Long> lowBlocks = mMetadataManager.getBlockIterator().getIterator(intersection.getSecond(), BlockOrder.REVERSE);
                while (lowBlocks.hasNext()) {
                    if (evictorView.isBlockEvictable(lowBlocks.next())) {
                        LOG.debug("Promotions needed from {} to {}", intersection.getSecond().tierAlias(), intersection.getFirst().tierAlias());
                        return TierManagementTaskType.PROMOTE;
                    }
                }
            }
        }
    }
    // No tier management task is required/allowed.
    return TierManagementTaskType.NONE;
}

Example 23 with StorageTier

use of alluxio.worker.block.meta.StorageTier in project alluxio by Alluxio.

the class BlockMetadataViewTest method getTierView.

/**
 * Tests the {@link BlockMetadataEvictorView#getTierView(String)} method.
 */
@Test
public void getTierView() {
    for (StorageTier tier : mMetaManager.getTiers()) {
        String tierAlias = tier.getTierAlias();
        StorageTierView tierView = mMetadataView.getTierView(tierAlias);
        assertEquals(tier.getTierAlias(), tierView.getTierViewAlias());
        assertEquals(tier.getTierOrdinal(), tierView.getTierViewOrdinal());
    }
}

Example 24 with StorageTier

use of alluxio.worker.block.meta.StorageTier in project alluxio by Alluxio.

the class BlockMetadataViewTest method getAvailableBytes.

/**
 * Tests the {@link BlockMetadataEvictorView#getAvailableBytes(BlockStoreLocation)} method.
 */
@Test
public void getAvailableBytes() {
    BlockStoreLocation location;
    // When location represents anyTier
    location = BlockStoreLocation.anyTier();
    assertEquals(mMetaManager.getAvailableBytes(location), mMetadataView.getAvailableBytes(location));
    // When location represents one particular tier
    for (StorageTier tier : mMetaManager.getTiers()) {
        String tierAlias = tier.getTierAlias();
        location = BlockStoreLocation.anyDirInTier(tierAlias);
        assertEquals(mMetaManager.getAvailableBytes(location), mMetadataView.getAvailableBytes(location));
        for (StorageDir dir : tier.getStorageDirs()) {
            // When location represents one particular dir
            location = dir.toBlockStoreLocation();
            assertEquals(mMetaManager.getAvailableBytes(location), mMetadataView.getAvailableBytes(location));
        }
    }
}

Example 25 with StorageTier

use of alluxio.worker.block.meta.StorageTier in project alluxio by Alluxio.

the class BlockMetadataViewTest method sameTierViewsBelow.

/**
 * Tests that {@link BlockMetadataEvictorView#getTierViewsBelow(String)} returns the same
 * TierViews as constructing by {@link BlockMetadataManager#getTiersBelow(String)}.
 */
@Test
public void sameTierViewsBelow() {
    String tierAlias = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getTierAlias();
    List<StorageTierView> tierViews1 = mMetadataView.getTierViewsBelow(tierAlias);
    // Do some operations on metadata
    StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL + 1).getDir(TEST_DIR);
    BlockMeta blockMeta = new DefaultBlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir);
    try {
        dir.addBlockMeta(blockMeta);
    } catch (Exception e) {
        e.printStackTrace();
    }
    List<StorageTier> tiers2 = mMetaManager.getTiersBelow(tierAlias);
    assertEquals(tierViews1.size(), tiers2.size());
    for (int i = 0; i < tierViews1.size(); i++) {
        assertSameTierView((StorageTierEvictorView) tierViews1.get(i), new StorageTierEvictorView(tiers2.get(i), mMetadataView));
    }
}