Examples with StorageDirEvictorView alluxio.worker.block.meta.StorageDirEvictorView used on opensource projects

Example 1 with StorageDirEvictorView

use of alluxio.worker.block.meta.StorageDirEvictorView in project alluxio by Alluxio.

the class BlockMetadataViewTest method assertSameTierView.

/**
 * Assert if two TierViews are the same by comparing their contents.
 */
private void assertSameTierView(StorageTierEvictorView tierView1, StorageTierEvictorView tierView2) {
    assertEquals(tierView1.getTierViewAlias(), tierView2.getTierViewAlias());
    assertEquals(tierView1.getTierViewOrdinal(), tierView2.getTierViewOrdinal());
    List<StorageDirView> dirViews1 = tierView1.getDirViews();
    List<StorageDirView> dirViews2 = tierView2.getDirViews();
    assertEquals(dirViews1.size(), dirViews2.size());
    for (int i = 0; i < dirViews1.size(); i++) {
        StorageDirEvictorView dirView1 = (StorageDirEvictorView) dirViews1.get(i);
        StorageDirEvictorView dirView2 = (StorageDirEvictorView) dirViews2.get(i);
        assertEquals(dirView1.getAvailableBytes(), dirView2.getAvailableBytes());
        assertEquals(dirView1.getCapacityBytes(), dirView2.getCapacityBytes());
        assertEquals(dirView1.getCommittedBytes(), dirView2.getCommittedBytes());
        assertEquals(dirView1.getDirViewIndex(), dirView2.getDirViewIndex());
        assertEquals(dirView1.getEvictableBlocks(), dirView2.getEvictableBlocks());
        assertEquals(dirView1.getEvitableBytes(), dirView2.getEvitableBytes());
    }
}

Example 2 with StorageDirEvictorView

use of alluxio.worker.block.meta.StorageDirEvictorView in project alluxio by Alluxio.

the class AbstractEvictor method cascadingEvict.

/**
 * A recursive implementation of cascading eviction.
 *
 * This method uses a specific eviction strategy to find blocks to evict in the requested
 * location. After eviction, one {@link alluxio.worker.block.meta.StorageDir} in the location has
 * the specific amount of free space. It then uses an allocation strategy to allocate space in the
 * next tier to move each evicted blocks. If the next tier fails to allocate space for the evicted
 * blocks, the next tier will continue to evict its blocks to free space.
 *
 * This method is only used in
 * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)}.
 *
 * @param bytesToBeAvailable bytes to be available after eviction
 * @param location target location to evict blocks from
 * @param plan the plan to be recursively updated, is empty when first called in
 *        {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)}
 * @param mode the eviction mode
 * @return the first {@link StorageDirEvictorView} in the range of location
 *         to evict/move bytes from, or null if there is no plan
 */
protected StorageDirEvictorView cascadingEvict(long bytesToBeAvailable, BlockStoreLocation location, EvictionPlan plan, Mode mode) {
    location = updateBlockStoreLocation(bytesToBeAvailable, location);
    // 1. If bytesToBeAvailable can already be satisfied without eviction, return the eligible
    // StorageDirView
    StorageDirEvictorView candidateDirView = (StorageDirEvictorView) EvictorUtils.selectDirWithRequestedSpace(bytesToBeAvailable, location, mMetadataView);
    if (candidateDirView != null) {
        return candidateDirView;
    }
    // 2. Iterate over blocks in order until we find a StorageDirEvictorView that is
    // in the range of location and can satisfy bytesToBeAvailable
    // after evicting its blocks iterated so far
    EvictionDirCandidates dirCandidates = new EvictionDirCandidates();
    Iterator<Long> it = getBlockIterator();
    while (it.hasNext() && dirCandidates.candidateSize() < bytesToBeAvailable) {
        long blockId = it.next();
        try {
            BlockMeta block = mMetadataView.getBlockMeta(blockId);
            if (block != null) {
                // might not present in this view
                if (block.getBlockLocation().belongsTo(location)) {
                    String tierAlias = block.getParentDir().getParentTier().getTierAlias();
                    int dirIndex = block.getParentDir().getDirIndex();
                    StorageDirView dirView = mMetadataView.getTierView(tierAlias).getDirView(dirIndex);
                    if (dirView != null) {
                        dirCandidates.add((StorageDirEvictorView) dirView, blockId, block.getBlockSize());
                    }
                }
            }
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Remove block {} from evictor cache because {}", blockId, e);
            it.remove();
            onRemoveBlockFromIterator(blockId);
        }
    }
    // 3. If there is no eligible StorageDirEvictorView, return null
    if (mode == Mode.GUARANTEED && dirCandidates.candidateSize() < bytesToBeAvailable) {
        return null;
    }
    // 4. cascading eviction: try to allocate space in the next tier to move candidate blocks
    // there. If allocation fails, the next tier will continue to evict its blocks to free space.
    // Blocks are only evicted from the last tier or it can not be moved to the next tier.
    candidateDirView = dirCandidates.candidateDir();
    if (candidateDirView == null) {
        return null;
    }
    List<Long> candidateBlocks = dirCandidates.candidateBlocks();
    StorageTierView nextTierView = mMetadataView.getNextTier(candidateDirView.getParentTierView());
    if (nextTierView == null) {
        // This is the last tier, evict all the blocks.
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mMetadataView.getBlockMeta(blockId);
                if (block != null) {
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    plan.toEvict().add(new Pair<>(blockId, candidateDirView.toBlockStoreLocation()));
                }
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    } else {
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mMetadataView.getBlockMeta(blockId);
                if (block == null) {
                    continue;
                }
                StorageDirEvictorView nextDirView = (StorageDirEvictorView) mAllocator.allocateBlockWithView(Sessions.MIGRATE_DATA_SESSION_ID, block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), mMetadataView, true);
                if (nextDirView == null) {
                    nextDirView = cascadingEvict(block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), plan, mode);
                }
                if (nextDirView == null) {
                    // If we failed to find a dir in the next tier to move this block, evict it and
                    // continue. Normally this should not happen.
                    plan.toEvict().add(new Pair<>(blockId, block.getBlockLocation()));
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    continue;
                }
                plan.toMove().add(BlockTransferInfo.createMove(block.getBlockLocation(), blockId, nextDirView.toBlockStoreLocation()));
                candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                nextDirView.markBlockMoveIn(blockId, block.getBlockSize());
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    }
    return candidateDirView;
}

Example 3 with StorageDirEvictorView

use of alluxio.worker.block.meta.StorageDirEvictorView in project alluxio by Alluxio.

the class AbstractEvictor method freeSpaceWithView.

@Override
public EvictionPlan freeSpaceWithView(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataEvictorView view, Mode mode) {
    mMetadataView = view;
    List<BlockTransferInfo> toMove = new ArrayList<>();
    List<Pair<Long, BlockStoreLocation>> toEvict = new ArrayList<>();
    EvictionPlan plan = new EvictionPlan(toMove, toEvict);
    StorageDirEvictorView candidateDir = cascadingEvict(bytesToBeAvailable, location, plan, mode);
    mMetadataView.clearBlockMarks();
    if (candidateDir == null) {
        return null;
    }
    return plan;
}

Example 4 with StorageDirEvictorView

use of alluxio.worker.block.meta.StorageDirEvictorView in project alluxio by Alluxio.

the class SwapRestoreTask method getBalancingTransfersList.

/**
 * @return the list of transfer in order to balance swap-space within tier
 */
private List<BlockTransferInfo> getBalancingTransfersList() {
    List<BlockTransferInfo> transferInfos = new LinkedList<>();
    for (StorageTierView tierView : mEvictorView.getTierViews()) {
        for (StorageDirView dirView : tierView.getDirViews()) {
            Iterator<Long> dirBlockIter = mMetadataManager.getBlockIterator().getIterator(new BlockStoreLocation(tierView.getTierViewAlias(), dirView.getDirViewIndex()), BlockOrder.NATURAL);
            while (dirBlockIter.hasNext() && dirView.getAvailableBytes() < dirView.getReservedBytes()) {
                long blockId = dirBlockIter.next();
                try {
                    BlockMeta movingOutBlock = mEvictorView.getBlockMeta(blockId);
                    if (movingOutBlock == null) {
                        LOG.debug("Block:{} exist but not available for balancing.", blockId);
                        continue;
                    }
                    // Find where to move the block.
                    StorageDirView dstDirView = null;
                    for (StorageDirView candidateDirView : tierView.getDirViews()) {
                        if (candidateDirView.getDirViewIndex() == dirView.getDirViewIndex()) {
                            continue;
                        }
                        if (candidateDirView.getAvailableBytes() - candidateDirView.getReservedBytes() >= movingOutBlock.getBlockSize()) {
                            dstDirView = candidateDirView;
                            break;
                        }
                    }
                    if (dstDirView == null) {
                        LOG.warn("Could not balance swap-restore space for location: {}", dirView.toBlockStoreLocation());
                        break;
                    }
                    // TODO(ggezer): Consider allowing evictions for this move.
                    transferInfos.add(BlockTransferInfo.createMove(movingOutBlock.getBlockLocation(), blockId, dstDirView.toBlockStoreLocation()));
                    // Account for moving-out blocks.
                    ((StorageDirEvictorView) dirView).markBlockMoveOut(blockId, movingOutBlock.getBlockSize());
                    // Account for moving-in blocks.
                    ((StorageDirEvictorView) dstDirView).markBlockMoveIn(blockId, movingOutBlock.getBlockSize());
                } catch (BlockDoesNotExistException e) {
                    LOG.warn("Failed to find metadata for block:{} during swap-restore balancing.", blockId);
                }
            }
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Generated {} balance transfers:\n ->{}", transferInfos.size(), transferInfos.stream().map(Object::toString).collect(Collectors.joining(",\n ->")));
    }
    return transferInfos;
}