Examples with StorageTierView alluxio.worker.block.meta.StorageTierView used on opensource projects

Example 1 with StorageTierView

use of alluxio.worker.block.meta.StorageTierView in project alluxio by Alluxio.

the class GreedyAllocator method allocateBlock.

/**
   * Allocates a block from the given block store location. The location can be a specific location,
   * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}.
   *
   * @param sessionId the id of session to apply for the block allocation
   * @param blockSize the size of block in bytes
   * @param location the location in block store
   * @return a {@link StorageDirView} in which to create the temp block meta if success, null
   *         otherwise
   * @throws IllegalArgumentException if block location is invalid
   */
private StorageDirView allocateBlock(long sessionId, long blockSize, BlockStoreLocation location) {
    Preconditions.checkNotNull(location);
    if (location.equals(BlockStoreLocation.anyTier())) {
        // and return a temp block meta from the first available dirview.
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getAvailableBytes() >= blockSize) {
                    return dirView;
                }
            }
        }
        return null;
    }
    String tierAlias = location.tierAlias();
    StorageTierView tierView = mManagerView.getTierView(tierAlias);
    if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
        // Loop over all dir views in the given tier
        for (StorageDirView dirView : tierView.getDirViews()) {
            if (dirView.getAvailableBytes() >= blockSize) {
                return dirView;
            }
        }
        return null;
    }
    int dirIndex = location.dir();
    StorageDirView dirView = tierView.getDirView(dirIndex);
    if (dirView.getAvailableBytes() >= blockSize) {
        return dirView;
    }
    return null;
}

Example 2 with StorageTierView

use of alluxio.worker.block.meta.StorageTierView in project alluxio by Alluxio.

the class MaxFreeAllocator method allocateBlock.

/**
   * Allocates a block from the given block store location. The location can be a specific location,
   * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}.
   *
   * @param sessionId the id of session to apply for the block allocation
   * @param blockSize the size of block in bytes
   * @param location the location in block store
   * @return a {@link StorageDirView} in which to create the temp block meta if success, null
   *         otherwise
   * @throws IllegalArgumentException if block location is invalid
   */
private StorageDirView allocateBlock(long sessionId, long blockSize, BlockStoreLocation location) {
    Preconditions.checkNotNull(location);
    StorageDirView candidateDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            candidateDirView = getCandidateDirInTier(tierView, blockSize);
            if (candidateDirView != null) {
                break;
            }
        }
    } else if (location.equals(BlockStoreLocation.anyDirInTier(location.tierAlias()))) {
        StorageTierView tierView = mManagerView.getTierView(location.tierAlias());
        candidateDirView = getCandidateDirInTier(tierView, blockSize);
    } else {
        StorageTierView tierView = mManagerView.getTierView(location.tierAlias());
        StorageDirView dirView = tierView.getDirView(location.dir());
        if (dirView.getAvailableBytes() >= blockSize) {
            candidateDirView = dirView;
        }
    }
    return candidateDirView;
}

Example 3 with StorageTierView

use of alluxio.worker.block.meta.StorageTierView in project alluxio by Alluxio.

the class AbstractEvictor method cascadingEvict.

/**
   * A recursive implementation of cascading eviction.
   *
   * This method uses a specific eviction strategy to find blocks to evict in the requested
   * location. After eviction, one {@link alluxio.worker.block.meta.StorageDir} in the location has
   * the specific amount of free space. It then uses an allocation strategy to allocate space in the
   * next tier to move each evicted blocks. If the next tier fails to allocate space for the evicted
   * blocks, the next tier will continue to evict its blocks to free space.
   *
   * This method is only used in
   * {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataManagerView)}.
   *
   * @param bytesToBeAvailable bytes to be available after eviction
   * @param location target location to evict blocks from
   * @param plan the plan to be recursively updated, is empty when first called in
   *        {@link #freeSpaceWithView(long, BlockStoreLocation, BlockMetadataManagerView)}
   * @return the first {@link StorageDirView} in the range of location to evict/move bytes from, or
   *         null if there is no plan
   */
protected StorageDirView cascadingEvict(long bytesToBeAvailable, BlockStoreLocation location, EvictionPlan plan) {
    location = updateBlockStoreLocation(bytesToBeAvailable, location);
    // 1. If bytesToBeAvailable can already be satisfied without eviction, return the eligible
    // StoargeDirView
    StorageDirView candidateDirView = EvictorUtils.selectDirWithRequestedSpace(bytesToBeAvailable, location, mManagerView);
    if (candidateDirView != null) {
        return candidateDirView;
    }
    // 2. Iterate over blocks in order until we find a StorageDirView that is in the range of
    // location and can satisfy bytesToBeAvailable after evicting its blocks iterated so far
    EvictionDirCandidates dirCandidates = new EvictionDirCandidates();
    Iterator<Long> it = getBlockIterator();
    while (it.hasNext() && dirCandidates.candidateSize() < bytesToBeAvailable) {
        long blockId = it.next();
        try {
            BlockMeta block = mManagerView.getBlockMeta(blockId);
            if (block != null) {
                // might not present in this view
                if (block.getBlockLocation().belongsTo(location)) {
                    String tierAlias = block.getParentDir().getParentTier().getTierAlias();
                    int dirIndex = block.getParentDir().getDirIndex();
                    dirCandidates.add(mManagerView.getTierView(tierAlias).getDirView(dirIndex), blockId, block.getBlockSize());
                }
            }
        } catch (BlockDoesNotExistException e) {
            LOG.warn("Remove block {} from evictor cache because {}", blockId, e);
            it.remove();
            onRemoveBlockFromIterator(blockId);
        }
    }
    // 3. If there is no eligible StorageDirView, return null
    if (dirCandidates.candidateSize() < bytesToBeAvailable) {
        return null;
    }
    // 4. cascading eviction: try to allocate space in the next tier to move candidate blocks
    // there. If allocation fails, the next tier will continue to evict its blocks to free space.
    // Blocks are only evicted from the last tier or it can not be moved to the next tier.
    candidateDirView = dirCandidates.candidateDir();
    List<Long> candidateBlocks = dirCandidates.candidateBlocks();
    StorageTierView nextTierView = mManagerView.getNextTier(candidateDirView.getParentTierView());
    if (nextTierView == null) {
        // This is the last tier, evict all the blocks.
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mManagerView.getBlockMeta(blockId);
                if (block != null) {
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    plan.toEvict().add(new Pair<>(blockId, candidateDirView.toBlockStoreLocation()));
                }
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    } else {
        for (Long blockId : candidateBlocks) {
            try {
                BlockMeta block = mManagerView.getBlockMeta(blockId);
                if (block == null) {
                    continue;
                }
                StorageDirView nextDirView = mAllocator.allocateBlockWithView(Sessions.MIGRATE_DATA_SESSION_ID, block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), mManagerView);
                if (nextDirView == null) {
                    nextDirView = cascadingEvict(block.getBlockSize(), BlockStoreLocation.anyDirInTier(nextTierView.getTierViewAlias()), plan);
                }
                if (nextDirView == null) {
                    // If we failed to find a dir in the next tier to move this block, evict it and
                    // continue. Normally this should not happen.
                    plan.toEvict().add(new Pair<>(blockId, block.getBlockLocation()));
                    candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                    continue;
                }
                plan.toMove().add(new BlockTransferInfo(blockId, block.getBlockLocation(), nextDirView.toBlockStoreLocation()));
                candidateDirView.markBlockMoveOut(blockId, block.getBlockSize());
                nextDirView.markBlockMoveIn(blockId, block.getBlockSize());
            } catch (BlockDoesNotExistException e) {
                continue;
            }
        }
    }
    return candidateDirView;
}

Example 4 with StorageTierView

use of alluxio.worker.block.meta.StorageTierView in project alluxio by Alluxio.

the class BlockMetadataManagerViewTest method sameTierView.

/**
   * Tests that {@code BlockMetadataManagerView.getTierView(tierAlias)} returns the same
   * TierView as {@code new StorageTierView(mMetadataManager.getTier(tierAlias), this)}.
   */
@Test
public void sameTierView() {
    String tierAlias = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getTierAlias();
    StorageTierView tierView1 = mMetaManagerView.getTierView(tierAlias);
    // Do some operations on metadata
    StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getDir(TEST_DIR);
    BlockMeta blockMeta = new BlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir);
    try {
        dir.addBlockMeta(blockMeta);
    } catch (Exception e) {
        e.printStackTrace();
    }
    StorageTierView tierView2 = new StorageTierView(mMetaManager.getTier(tierAlias), mMetaManagerView);
    assertSameTierView(tierView1, tierView2);
}

Example 5 with StorageTierView

use of alluxio.worker.block.meta.StorageTierView in project alluxio by Alluxio.

the class EvictorUtils method getDirWithMaxFreeSpace.

/**
   * Gets {@link StorageDirView} with max free space.
   *
   * @param bytesToBeAvailable space size to be requested
   * @param location location that the space will be allocated in
   * @param mManagerView a view of block metadata information
   * @return the {@link StorageDirView} selected
   */
public static StorageDirView getDirWithMaxFreeSpace(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataManagerView mManagerView) {
    long maxFreeSize = -1;
    StorageDirView selectedDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                    selectedDirView = dirView;
                    maxFreeSize = dirView.getAvailableBytes();
                }
            }
        }
    } else {
        String tierAlias = location.tierAlias();
        StorageTierView tierView = mManagerView.getTierView(tierAlias);
        if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                    selectedDirView = dirView;
                    maxFreeSize = dirView.getAvailableBytes();
                }
            }
        } else {
            int dirIndex = location.dir();
            StorageDirView dirView = tierView.getDirView(dirIndex);
            if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                selectedDirView = dirView;
            }
        }
    }
    return selectedDirView;
}