Examples with StorageDirView alluxio.worker.block.meta.StorageDirView used on opensource projects

Example 1 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class GreedyAllocator method allocateBlock.

/**
   * Allocates a block from the given block store location. The location can be a specific location,
   * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}.
   *
   * @param sessionId the id of session to apply for the block allocation
   * @param blockSize the size of block in bytes
   * @param location the location in block store
   * @return a {@link StorageDirView} in which to create the temp block meta if success, null
   *         otherwise
   * @throws IllegalArgumentException if block location is invalid
   */
private StorageDirView allocateBlock(long sessionId, long blockSize, BlockStoreLocation location) {
    Preconditions.checkNotNull(location);
    if (location.equals(BlockStoreLocation.anyTier())) {
        // and return a temp block meta from the first available dirview.
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getAvailableBytes() >= blockSize) {
                    return dirView;
                }
            }
        }
        return null;
    }
    String tierAlias = location.tierAlias();
    StorageTierView tierView = mManagerView.getTierView(tierAlias);
    if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
        // Loop over all dir views in the given tier
        for (StorageDirView dirView : tierView.getDirViews()) {
            if (dirView.getAvailableBytes() >= blockSize) {
                return dirView;
            }
        }
        return null;
    }
    int dirIndex = location.dir();
    StorageDirView dirView = tierView.getDirView(dirIndex);
    if (dirView.getAvailableBytes() >= blockSize) {
        return dirView;
    }
    return null;
}

Example 2 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class MaxFreeAllocator method allocateBlock.

/**
   * Allocates a block from the given block store location. The location can be a specific location,
   * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}.
   *
   * @param sessionId the id of session to apply for the block allocation
   * @param blockSize the size of block in bytes
   * @param location the location in block store
   * @return a {@link StorageDirView} in which to create the temp block meta if success, null
   *         otherwise
   * @throws IllegalArgumentException if block location is invalid
   */
private StorageDirView allocateBlock(long sessionId, long blockSize, BlockStoreLocation location) {
    Preconditions.checkNotNull(location);
    StorageDirView candidateDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            candidateDirView = getCandidateDirInTier(tierView, blockSize);
            if (candidateDirView != null) {
                break;
            }
        }
    } else if (location.equals(BlockStoreLocation.anyDirInTier(location.tierAlias()))) {
        StorageTierView tierView = mManagerView.getTierView(location.tierAlias());
        candidateDirView = getCandidateDirInTier(tierView, blockSize);
    } else {
        StorageTierView tierView = mManagerView.getTierView(location.tierAlias());
        StorageDirView dirView = tierView.getDirView(location.dir());
        if (dirView.getAvailableBytes() >= blockSize) {
            candidateDirView = dirView;
        }
    }
    return candidateDirView;
}

Example 3 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class MaxFreeAllocator method getCandidateDirInTier.

/**
   * Finds a directory view in a tier view that has max free space and is able to store the block.
   *
   * @param tierView the storage tier view
   * @param blockSize the size of block in bytes
   * @return the storage directory view if found, null otherwise
   */
private StorageDirView getCandidateDirInTier(StorageTierView tierView, long blockSize) {
    StorageDirView candidateDirView = null;
    long maxFreeBytes = blockSize - 1;
    for (StorageDirView dirView : tierView.getDirViews()) {
        if (dirView.getAvailableBytes() > maxFreeBytes) {
            maxFreeBytes = dirView.getAvailableBytes();
            candidateDirView = dirView;
        }
    }
    return candidateDirView;
}

Example 4 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class AbstractEvictor method cascadingEvict.

/**
   * A recursive implementation of cascading eviction.
   *
   * This method uses a specific eviction strategy to find blocks to evict in the requested
   * location. After eviction, one {@link alluxio.worker.block.meta.StorageDir} in the location has
   * the specific amount of free space. It then uses an allocation strategy to allocate space in the
   * next tier to move each evicted blocks. If the next tier fails to allocate space for the evicted
   * blocks, the next tier will continue to evict its blocks to free space.
   *
   * This method is only used in
   * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataManagerView)}.
   *
   * @param bytesToBeAvailable bytes to be available after eviction
   * @param location target location to evict blocks from
   * @param plan the plan to be recursively updated, is empty when first called in
   *        {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataManagerView)}
   * @return the first {@link StorageDirView} in the range of location to evict/move bytes from, or
   *         null if there is no plan
   */
protected StorageDirView cascadingEvict(long bytesToBeAvailable, BlockStoreLocation location, EvictionPlan plan) {
    location = updateBlockStoreLocation(bytesToBeAvailable, location);
    // 1. If bytesToBeAvailable can already be satisfied without eviction, return the eligible
    // StoargeDirView
    StorageDirView candidateDirView = EvictorUtils.selectDirWithRequestedSpace(bytesToBeAvailable, location, mManagerView);
    if (candidateDirView != null) {
        return candidateDirView;
    }
    // 2. Iterate over blocks in order until we find a StorageDirView that is in the range of
    // location and can satisfy bytesToBeAvailable after evicting its blocks iterated so far
    EvictionDirCandidates dirCandidates = new EvictionDirCandidates();
    Iterator<Long> it = getBlockIterator();
    while (it.hasNext() && dirCandidates.candidateSize() < bytesToBeAvailable) {
        long blockId = it.next();
        try {
            BlockMeta block = mManagerView.getBlockMeta(blockId);
            if (block != null) {
                // might not present in this view
                if (block.getBlockLocation().belongsTo(location)) {
                    String tierAlias = block.getParentDir().getParentTier().getTierAlias();
                    int dirIndex = block.getParentDir().getDirIndex();
                    dirCandidates.add(mManagerView.getTierView(tierAlias).getDirView(dirIndex), blockId, block.getBlockSize());
                }
            }
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Remove block {} from evictor cache because {}", blockId, e);
            it.remove();
            onRemoveBlockFromIterator(blockId);
        }
    }
    // 3. If there is no eligible StorageDirView, return null
    if (dirCandidates.candidateSize() < bytesToBeAvailable) {
        return null;
    }
    // 4. cascading eviction: try to allocate space in the next tier to move candidate blocks
    // there. If allocation fails, the next tier will continue to evict its blocks to free space.
    // Blocks are only evicted from the last tier or it can not be moved to the next tier.
    candidateDirView = dirCandidates.candidateDir();
    List<Long> candidateBlocks = dirCandidates.candidateBlocks();
    StorageTierView nextTierView = mManagerView.getNextTier(candidateDirView.getParentTierView());
    if (nextTierView == null) {
        // This is the last tier, evict all the blocks.
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mManagerView.getBlockMeta(blockId);
                if (block != null) {
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    plan.toEvict().add(new Pair<>(blockId, candidateDirView.toBlockStoreLocation()));
                }
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    } else {
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mManagerView.getBlockMeta(blockId);
                if (block == null) {
                    continue;
                }
                StorageDirView nextDirView = mAllocator.allocateBlockWithView(Sessions.MIGRATE_DATA_SESSION_ID, block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), mManagerView);
                if (nextDirView == null) {
                    nextDirView = cascadingEvict(block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), plan);
                }
                if (nextDirView == null) {
                    // If we failed to find a dir in the next tier to move this block, evict it and
                    // continue. Normally this should not happen.
                    plan.toEvict().add(new Pair<>(blockId, block.getBlockLocation()));
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    continue;
                }
                plan.toMove().add(new BlockTransferInfo(blockId, block.getBlockLocation(), nextDirView.toBlockStoreLocation()));
                candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                nextDirView.markBlockMoveIn(blockId, block.getBlockSize());
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    }
    return candidateDirView;
}

Example 5 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class AbstractEvictor method freeSpaceWithView.

@Override
public EvictionPlan freeSpaceWithView(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataManagerView view) {
    mManagerView = view;
    List<BlockTransferInfo> toMove = new ArrayList<>();
    List<Pair<Long, BlockStoreLocation>> toEvict = new ArrayList<>();
    EvictionPlan plan = new EvictionPlan(toMove, toEvict);
    StorageDirView candidateDir = cascadingEvict(bytesToBeAvailable, location, plan);
    mManagerView.clearBlockMarks();
    if (candidateDir == null) {
        return null;
    }
    return plan;
}