Examples with StorageDirView alluxio.worker.block.meta.StorageDirView used on opensource projects

Example 16 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class EvictorUtils method getDirWithMaxFreeSpace.

/**
   * Gets {@link StorageDirView} with max free space.
   *
   * @param bytesToBeAvailable space size to be requested
   * @param location location that the space will be allocated in
   * @param mManagerView a view of block metadata information
   * @return the {@link StorageDirView} selected
   */
public static StorageDirView getDirWithMaxFreeSpace(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataManagerView mManagerView) {
    long maxFreeSize = -1;
    StorageDirView selectedDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                    selectedDirView = dirView;
                    maxFreeSize = dirView.getAvailableBytes();
                }
            }
        }
    } else {
        String tierAlias = location.tierAlias();
        StorageTierView tierView = mManagerView.getTierView(tierAlias);
        if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                    selectedDirView = dirView;
                    maxFreeSize = dirView.getAvailableBytes();
                }
            }
        } else {
            int dirIndex = location.dir();
            StorageDirView dirView = tierView.getDirView(dirIndex);
            if (dirView.getCommittedBytes() + dirView.getAvailableBytes() >= bytesToBeAvailable && dirView.getAvailableBytes() > maxFreeSize) {
                selectedDirView = dirView;
            }
        }
    }
    return selectedDirView;
}

Example 17 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class EvictorUtils method selectDirWithRequestedSpace.

/**
   * Finds a directory in the given location range with capacity upwards of the given bound.
   *
   * @param bytesToBeAvailable the capacity bound
   * @param location the location range
   * @param mManagerView the storage manager view
   * @return a {@link StorageDirView} in the range of location that already has availableBytes
   *         larger than bytesToBeAvailable, otherwise null
   */
public static StorageDirView selectDirWithRequestedSpace(long bytesToBeAvailable, BlockStoreLocation location, BlockMetadataManagerView mManagerView) {
    if (location.equals(BlockStoreLocation.anyTier())) {
        for (StorageTierView tierView : mManagerView.getTierViews()) {
            for (StorageDirView dirView : tierView.getDirViews()) {
                if (dirView.getAvailableBytes() >= bytesToBeAvailable) {
                    return dirView;
                }
            }
        }
        return null;
    }
    String tierAlias = location.tierAlias();
    StorageTierView tierView = mManagerView.getTierView(tierAlias);
    if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
        for (StorageDirView dirView : tierView.getDirViews()) {
            if (dirView.getAvailableBytes() >= bytesToBeAvailable) {
                return dirView;
            }
        }
        return null;
    }
    StorageDirView dirView = tierView.getDirView(location.dir());
    return (dirView.getAvailableBytes() >= bytesToBeAvailable) ? dirView : null;
}

Example 18 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class GreedyEvictor method freeSpaceWithView.

@Override
public EvictionPlan freeSpaceWithView(long availableBytes, BlockStoreLocation location, BlockMetadataManagerView view) {
    Preconditions.checkNotNull(location);
    Preconditions.checkNotNull(view);
    // 1. Select a StorageDirView that has enough capacity for required bytes.
    StorageDirView selectedDirView = null;
    if (location.equals(BlockStoreLocation.anyTier())) {
        selectedDirView = selectEvictableDirFromAnyTier(view, availableBytes);
    } else {
        String tierAlias = location.tierAlias();
        StorageTierView tierView = view.getTierView(tierAlias);
        if (location.equals(BlockStoreLocation.anyDirInTier(tierAlias))) {
            selectedDirView = selectEvictableDirFromTier(tierView, availableBytes);
        } else {
            int dirIndex = location.dir();
            StorageDirView dir = tierView.getDirView(dirIndex);
            if (canEvictBlocksFromDir(dir, availableBytes)) {
                selectedDirView = dir;
            }
        }
    }
    if (selectedDirView == null) {
        LOG.error("Failed to freeSpace: No StorageDirView has enough capacity of {} bytes", availableBytes);
        return null;
    }
    // 2. Check if the selected StorageDirView already has enough space.
    List<BlockTransferInfo> toTransfer = new ArrayList<>();
    List<Pair<Long, BlockStoreLocation>> toEvict = new ArrayList<>();
    long bytesAvailableInDir = selectedDirView.getAvailableBytes();
    if (bytesAvailableInDir >= availableBytes) {
        // No need to evict anything, return an eviction plan with empty instructions.
        return new EvictionPlan(toTransfer, toEvict);
    }
    // 3. Collect victim blocks from the selected StorageDirView. They could either be evicted or
    // moved.
    List<BlockMeta> victimBlocks = new ArrayList<>();
    for (BlockMeta block : selectedDirView.getEvictableBlocks()) {
        victimBlocks.add(block);
        bytesAvailableInDir += block.getBlockSize();
        if (bytesAvailableInDir >= availableBytes) {
            break;
        }
    }
    // 4. Make best effort to transfer victim blocks to lower tiers rather than evict them.
    Map<StorageDirView, Long> pendingBytesInDir = new HashMap<>();
    for (BlockMeta block : victimBlocks) {
        // TODO(qifan): Should avoid calling getParentDir.
        String fromTierAlias = block.getParentDir().getParentTier().getTierAlias();
        List<StorageTierView> candidateTiers = view.getTierViewsBelow(fromTierAlias);
        StorageDirView dstDir = selectAvailableDir(block, candidateTiers, pendingBytesInDir);
        if (dstDir == null) {
            // Not possible to transfer
            toEvict.add(new Pair<>(block.getBlockId(), block.getBlockLocation()));
        } else {
            StorageTierView dstTier = dstDir.getParentTierView();
            toTransfer.add(new BlockTransferInfo(block.getBlockId(), block.getBlockLocation(), new BlockStoreLocation(dstTier.getTierViewAlias(), dstDir.getDirViewIndex())));
            if (pendingBytesInDir.containsKey(dstDir)) {
                pendingBytesInDir.put(dstDir, pendingBytesInDir.get(dstDir) + block.getBlockSize());
            } else {
                pendingBytesInDir.put(dstDir, block.getBlockSize());
            }
        }
    }
    return new EvictionPlan(toTransfer, toEvict);
}

Example 19 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class BlockMetadataManagerViewTest method assertSameTierView.

/**
   * Assert if two TierViews are the same by comparing their contents.
   */
private void assertSameTierView(StorageTierView tierView1, StorageTierView tierView2) {
    Assert.assertEquals(tierView1.getTierViewAlias(), tierView2.getTierViewAlias());
    Assert.assertEquals(tierView1.getTierViewOrdinal(), tierView2.getTierViewOrdinal());
    List<StorageDirView> dirViews1 = tierView1.getDirViews();
    List<StorageDirView> dirViews2 = tierView2.getDirViews();
    Assert.assertEquals(dirViews1.size(), dirViews2.size());
    for (int i = 0; i < dirViews1.size(); i++) {
        StorageDirView dirView1 = dirViews1.get(i);
        StorageDirView dirView2 = dirViews2.get(i);
        Assert.assertEquals(dirView1.getAvailableBytes(), dirView2.getAvailableBytes());
        Assert.assertEquals(dirView1.getCapacityBytes(), dirView2.getCapacityBytes());
        Assert.assertEquals(dirView1.getCommittedBytes(), dirView2.getCommittedBytes());
        Assert.assertEquals(dirView1.getDirViewIndex(), dirView2.getDirViewIndex());
        Assert.assertEquals(dirView1.getEvictableBlocks(), dirView2.getEvictableBlocks());
        Assert.assertEquals(dirView1.getEvitableBytes(), dirView2.getEvitableBytes());
    }
}

Example 20 with StorageDirView

use of alluxio.worker.block.meta.StorageDirView in project alluxio by Alluxio.

the class RoundRobinAllocator method allocateBlock.

/**
   * Allocates a block from the given block store location. The location can be a specific location,
   * or {@link BlockStoreLocation#anyTier()} or {@link BlockStoreLocation#anyDirInTier(String)}.
   *
   * @param sessionId the id of session to apply for the block allocation
   * @param blockSize the size of block in bytes
   * @param location the location in block store
   * @return a {@link StorageDirView} in which to create the temp block meta if success, null
   *         otherwise
   * @throws IllegalArgumentException if block location is invalid
   */
private StorageDirView allocateBlock(long sessionId, long blockSize, BlockStoreLocation location) {
    Preconditions.checkNotNull(location);
    if (location.equals(BlockStoreLocation.anyTier())) {
        // always starting from the first tier
        int tierIndex = 0;
        for (int i = 0; i < mManagerView.getTierViews().size(); i++) {
            StorageTierView tierView = mManagerView.getTierViews().get(tierIndex);
            int dirViewIndex = getNextAvailDirInTier(tierView, blockSize);
            if (dirViewIndex >= 0) {
                mTierAliasToLastDirMap.put(tierView.getTierViewAlias(), dirViewIndex);
                return tierView.getDirView(dirViewIndex);
            } else {
                // we didn't find one in this tier, go to next tier
                tierIndex++;
            }
        }
    } else if (location.equals(BlockStoreLocation.anyDirInTier(location.tierAlias()))) {
        StorageTierView tierView = mManagerView.getTierView(location.tierAlias());
        int dirViewIndex = getNextAvailDirInTier(tierView, blockSize);
        if (dirViewIndex >= 0) {
            mTierAliasToLastDirMap.put(tierView.getTierViewAlias(), dirViewIndex);
            return tierView.getDirView(dirViewIndex);
        }
    } else {
        StorageTierView tierView = mManagerView.getTierView(location.tierAlias());
        StorageDirView dirView = tierView.getDirView(location.dir());
        if (dirView.getAvailableBytes() >= blockSize) {
            return dirView;
        }
    }
    return null;
}