Examples with StorageDirView alluxio.worker.block.meta.StorageDirView used on opensource projects

Example 26 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class MaxFreeAllocator method getCandidateDirInTier.

/**
 * Finds a directory view in a tier view that has max free space and is able to store the block.
 *
 * @param tierView the storage tier view
 * @param blockSize the size of block in bytes
 * @param mediumType the medium type that must match
 * @return the storage directory view if found, null otherwise
 */
private StorageDirView getCandidateDirInTier(StorageTierView tierView, long blockSize, String mediumType) {
    StorageDirView candidateDirView = null;
    long maxFreeBytes = blockSize - 1;
    for (StorageDirView dirView : tierView.getDirViews()) {
        if ((mediumType.equals(BlockStoreLocation.ANY_MEDIUM) || dirView.getMediumType().equals(mediumType)) && dirView.getAvailableBytes() > maxFreeBytes) {
            maxFreeBytes = dirView.getAvailableBytes();
            candidateDirView = dirView;
        }
    }
    return candidateDirView;
}

Example 27 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class RoundRobinAllocator method allocateBlock.

/**
 * Allocates a block from the given block store location. The location can be a specific location,
 * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}.
 *
 * @param sessionId the id of session to apply for the block allocation
 * @param blockSize the size of block in bytes
 * @param location the location in block store
 * @return a {@link StorageDirView} in which to create the temp block meta if success,
 *         null otherwise
 * @throws IllegalArgumentException if block location is invalid
 */
@Nullable
private StorageDirView allocateBlock(long sessionId, long blockSize, BlockStoreLocation location, boolean skipReview) {
    Preconditions.checkNotNull(location, "location");
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (int i = 0; i < mMetadataView.getTierViews().size(); i++) {
            StorageTierView tierView = mMetadataView.getTierViews().get(i);
            // The review logic is handled in getNextAvailDirInTier
            int dirViewIndex = getNextAvailDirInTier(tierView, blockSize, BlockStoreLocation.ANY_MEDIUM, skipReview);
            if (dirViewIndex >= 0) {
                mTierAliasToLastDirMap.put(tierView.getTierViewAlias(), dirViewIndex);
                return tierView.getDirView(dirViewIndex);
            }
        }
    } else if (location.equals(BlockStoreLocation.anyDirInTier(location.tierAlias()))) {
        StorageTierView tierView = mMetadataView.getTierView(location.tierAlias());
        // The review logic is handled in getNextAvailDirInTier
        int dirViewIndex = getNextAvailDirInTier(tierView, blockSize, BlockStoreLocation.ANY_MEDIUM, skipReview);
        if (dirViewIndex >= 0) {
            mTierAliasToLastDirMap.put(tierView.getTierViewAlias(), dirViewIndex);
            return tierView.getDirView(dirViewIndex);
        }
    } else if (location.equals(BlockStoreLocation.anyDirInAnyTierWithMedium(location.mediumType()))) {
        for (int i = 0; i < mMetadataView.getTierViews().size(); i++) {
            StorageTierView tierView = mMetadataView.getTierViews().get(i);
            // The review logic is handled in getNextAvailDirInTier
            int dirViewIndex = getNextAvailDirInTier(tierView, blockSize, location.mediumType(), skipReview);
            if (dirViewIndex >= 0) {
                mTierAliasToLastDirMap.put(tierView.getTierViewAlias(), dirViewIndex);
                return tierView.getDirView(dirViewIndex);
            }
        }
    } else {
        // For allocation in a specific directory, we are not checking the reviewer,
        // because we do not want the reviewer to reject it.
        StorageTierView tierView = mMetadataView.getTierView(location.tierAlias());
        StorageDirView dirView = tierView.getDirView(location.dir());
        if (dirView != null && dirView.getAvailableBytes() >= blockSize) {
            return dirView;
        }
    }
    return null;
}

Example 28 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class AbstractEvictor method cascadingEvict.

/**
 * A recursive implementation of cascading eviction.
 *
 * This method uses a specific eviction strategy to find blocks to evict in the requested
 * location. After eviction, one {@link alluxio.worker.block.meta.StorageDir} in the location has
 * the specific amount of free space. It then uses an allocation strategy to allocate space in the
 * next tier to move each evicted blocks. If the next tier fails to allocate space for the evicted
 * blocks, the next tier will continue to evict its blocks to free space.
 *
 * This method is only used in
 * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)}.
 *
 * @param bytesToBeAvailable bytes to be available after eviction
 * @param location target location to evict blocks from
 * @param plan the plan to be recursively updated, is empty when first called in
 *        {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataEvictorView)}
 * @param mode the eviction mode
 * @return the first {@link StorageDirEvictorView} in the range of location
 *         to evict/move bytes from, or null if there is no plan
 */
protected StorageDirEvictorView cascadingEvict(long bytesToBeAvailable, BlockStoreLocation location, EvictionPlan plan, Mode mode) {
    location = updateBlockStoreLocation(bytesToBeAvailable, location);
    // 1. If bytesToBeAvailable can already be satisfied without eviction, return the eligible
    // StorageDirView
    StorageDirEvictorView candidateDirView = (StorageDirEvictorView) EvictorUtils.selectDirWithRequestedSpace(bytesToBeAvailable, location, mMetadataView);
    if (candidateDirView != null) {
        return candidateDirView;
    }
    // 2. Iterate over blocks in order until we find a StorageDirEvictorView that is
    // in the range of location and can satisfy bytesToBeAvailable
    // after evicting its blocks iterated so far
    EvictionDirCandidates dirCandidates = new EvictionDirCandidates();
    Iterator<Long> it = getBlockIterator();
    while (it.hasNext() && dirCandidates.candidateSize() < bytesToBeAvailable) {
        long blockId = it.next();
        try {
            BlockMeta block = mMetadataView.getBlockMeta(blockId);
            if (block != null) {
                // might not present in this view
                if (block.getBlockLocation().belongsTo(location)) {
                    String tierAlias = block.getParentDir().getParentTier().getTierAlias();
                    int dirIndex = block.getParentDir().getDirIndex();
                    StorageDirView dirView = mMetadataView.getTierView(tierAlias).getDirView(dirIndex);
                    if (dirView != null) {
                        dirCandidates.add((StorageDirEvictorView) dirView, blockId, block.getBlockSize());
                    }
                }
            }
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Remove block {} from evictor cache because {}", blockId, e);
            it.remove();
            onRemoveBlockFromIterator(blockId);
        }
    }
    // 3. If there is no eligible StorageDirEvictorView, return null
    if (mode == Mode.GUARANTEED && dirCandidates.candidateSize() < bytesToBeAvailable) {
        return null;
    }
    // 4. cascading eviction: try to allocate space in the next tier to move candidate blocks
    // there. If allocation fails, the next tier will continue to evict its blocks to free space.
    // Blocks are only evicted from the last tier or it can not be moved to the next tier.
    candidateDirView = dirCandidates.candidateDir();
    if (candidateDirView == null) {
        return null;
    }
    List<Long> candidateBlocks = dirCandidates.candidateBlocks();
    StorageTierView nextTierView = mMetadataView.getNextTier(candidateDirView.getParentTierView());
    if (nextTierView == null) {
        // This is the last tier, evict all the blocks.
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mMetadataView.getBlockMeta(blockId);
                if (block != null) {
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    plan.toEvict().add(new Pair<>(blockId, candidateDirView.toBlockStoreLocation()));
                }
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    } else {
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mMetadataView.getBlockMeta(blockId);
                if (block == null) {
                    continue;
                }
                StorageDirEvictorView nextDirView = (StorageDirEvictorView) mAllocator.allocateBlockWithView(Sessions.MIGRATE_DATA_SESSION_ID, block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), mMetadataView, true);
                if (nextDirView == null) {
                    nextDirView = cascadingEvict(block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), plan, mode);
                }
                if (nextDirView == null) {
                    // If we failed to find a dir in the next tier to move this block, evict it and
                    // continue. Normally this should not happen.
                    plan.toEvict().add(new Pair<>(blockId, block.getBlockLocation()));
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    continue;
                }
                plan.toMove().add(BlockTransferInfo.createMove(block.getBlockLocation(), blockId, nextDirView.toBlockStoreLocation()));
                candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                nextDirView.markBlockMoveIn(blockId, block.getBlockSize());
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    }
    return candidateDirView;
}

Example 29 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class EvictorUtils method selectDirWithRequestedSpace.

/**
 * Finds a directory in the given location range with capacity upwards of the given bound.
 *
 * @param bytesToBeAvailable the capacity bound
 * @param location the location range
 * @param mManagerView the storage manager view
 * @return a {@link StorageDirView} in the range of location that already
 *         has availableBytes larger than bytesToBeAvailable, otherwise null
 */
@Nullable
public static StorageDirView selectDirWithRequestedSpace(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataEvictorView mManagerView) {
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getAvailableBytes() >= bytesToBeAvailable) {
                    return dirView;
                }
            }
        }
        return null;
    }
    String tierAlias = location.tierAlias();
    StorageTierView tierView = mManagerView.getTierView(tierAlias);
    if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
        for (StorageDirView dirView : tierView.getDirViews()) {
            if (dirView.getAvailableBytes() >= bytesToBeAvailable) {
                return dirView;
            }
        }
        return null;
    }
    StorageDirView dirView = tierView.getDirView(location.dir());
    return (dirView != null && dirView.getAvailableBytes() >= bytesToBeAvailable) ? dirView : null;
}

Example 30 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class EvictorUtils method getDirWithMaxFreeSpace.

/**
 * Gets {@link StorageDirView} with max free space.
 *
 * @param bytesToBeAvailable space size to be requested
 * @param location location that the space will be allocated in
 * @param metadataView a view of block metadata information
 * @return the {@link StorageDirView} selected
 */
public static StorageDirView getDirWithMaxFreeSpace(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataEvictorView metadataView) {
    long maxFreeSize = -1;
    StorageDirView selectedDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (StorageTierView tierView : metadataView.getTierViews()) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                    selectedDirView = dirView;
                    maxFreeSize = dirView.getAvailableBytes();
                }
            }
        }
    } else {
        String tierAlias = location.tierAlias();
        StorageTierView tierView = metadataView.getTierView(tierAlias);
        if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                    selectedDirView = dirView;
                    maxFreeSize = dirView.getAvailableBytes();
                }
            }
        } else {
            int dirIndex = location.dir();
            StorageDirView dirView = tierView.getDirView(dirIndex);
            if (dirView != null && dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                selectedDirView = dirView;
            }
        }
    }
    return selectedDirView;
}