Examples with BlockMeta alluxio.worker.block.meta.BlockMeta used on opensource projects

Example 6 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class EvictorTestUtils method validCascadingPlan.

/**
   * Checks whether the plan of a cascading evictor is valid.
   *
   * A cascading evictor will try to free space by recursively moving blocks to next 1 tier and
   * evict blocks only in the bottom tier.
   *
   * The plan is invalid when the requested space can not be satisfied or lower level of tiers do
   * not have enough space to hold blocks moved from higher level of tiers.
   *
   * @param bytesToBeAvailable requested bytes to be available after eviction
   * @param plan the eviction plan, should not be empty
   * @param metaManager the metadata manager
   * @return true if the above requirements are satisfied, otherwise false
   * @throws BlockDoesNotExistException if a block for which metadata cannot be found is encountered
   */
// TODO(bin): Add a unit test for this method.
public static boolean validCascadingPlan(long bytesToBeAvailable, EvictionPlan plan, BlockMetadataManager metaManager) throws BlockDoesNotExistException {
    // reassure the plan is feasible: enough free space to satisfy bytesToBeAvailable, and enough
    // space in lower tier to move blocks in upper tier there
    // Map from dir to a pair of bytes to be available in this dir and bytes to move into this dir
    // after the plan taking action
    Map<StorageDir, Pair<Long, Long>> spaceInfoInDir = new HashMap<>();
    for (Pair<Long, BlockStoreLocation> blockInfo : plan.toEvict()) {
        BlockMeta block = metaManager.getBlockMeta(blockInfo.getFirst());
        StorageDir dir = block.getParentDir();
        if (spaceInfoInDir.containsKey(dir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(dir);
            spaceInfo.setFirst(spaceInfo.getFirst() + block.getBlockSize());
        } else {
            spaceInfoInDir.put(dir, new Pair<>(dir.getAvailableBytes() + block.getBlockSize(), 0L));
        }
    }
    for (BlockTransferInfo move : plan.toMove()) {
        long blockId = move.getBlockId();
        BlockMeta block = metaManager.getBlockMeta(blockId);
        long blockSize = block.getBlockSize();
        StorageDir srcDir = block.getParentDir();
        StorageDir destDir = metaManager.getDir(move.getDstLocation());
        if (spaceInfoInDir.containsKey(srcDir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(srcDir);
            spaceInfo.setFirst(spaceInfo.getFirst() + blockSize);
        } else {
            spaceInfoInDir.put(srcDir, new Pair<>(srcDir.getAvailableBytes() + blockSize, 0L));
        }
        if (spaceInfoInDir.containsKey(destDir)) {
            Pair<Long, Long> spaceInfo = spaceInfoInDir.get(destDir);
            spaceInfo.setSecond(spaceInfo.getSecond() + blockSize);
        } else {
            spaceInfoInDir.put(destDir, new Pair<>(destDir.getAvailableBytes(), blockSize));
        }
    }
    // the top tier among all tiers where blocks in the plan reside in
    int topTierOrdinal = Integer.MAX_VALUE;
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        topTierOrdinal = Math.min(topTierOrdinal, dir.getParentTier().getTierOrdinal());
    }
    // maximum bytes to be available in a dir in the top tier
    long maxSpace = Long.MIN_VALUE;
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        if (dir.getParentTier().getTierOrdinal() == topTierOrdinal) {
            Pair<Long, Long> space = spaceInfoInDir.get(dir);
            maxSpace = Math.max(maxSpace, space.getFirst() - space.getSecond());
        }
    }
    if (maxSpace < bytesToBeAvailable) {
        // plan is invalid because requested space can not be satisfied in the top tier
        return false;
    }
    for (StorageDir dir : spaceInfoInDir.keySet()) {
        Pair<Long, Long> spaceInfo = spaceInfoInDir.get(dir);
        if (spaceInfo.getFirst() < spaceInfo.getSecond()) {
            // to be moved into this dir
            return false;
        }
    }
    return true;
}

Example 7 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManagerTest method moveBlockMetaDiffDir.

/**
   * Tests that an exception is thrown in the
   * {@link BlockMetadataManager#moveBlockMeta(BlockMeta, TempBlockMeta)} method when trying to move
   * a block to a not committed block meta.
   */
@Test
public void moveBlockMetaDiffDir() throws Exception {
    // create and add two temp block metas with different dirs in the same HDD tier
    StorageDir dir1 = mMetaManager.getTier("HDD").getDir(0);
    StorageDir dir2 = mMetaManager.getTier("HDD").getDir(1);
    TempBlockMeta tempBlockMeta1 = new TempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE, dir1);
    TempBlockMeta tempBlockMeta2 = new TempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID2, TEST_BLOCK_SIZE, dir2);
    mMetaManager.addTempBlockMeta(tempBlockMeta1);
    mMetaManager.addTempBlockMeta(tempBlockMeta2);
    // commit the first temp block meta
    mMetaManager.commitTempBlockMeta(tempBlockMeta1);
    BlockMeta blockMeta = mMetaManager.getBlockMeta(TEST_TEMP_BLOCK_ID);
    mMetaManager.moveBlockMeta(blockMeta, tempBlockMeta2);
    // make sure that the dst tempBlockMeta has been removed from the dir2
    mThrown.expect(BlockDoesNotExistException.class);
    mThrown.expectMessage(ExceptionMessage.TEMP_BLOCK_META_NOT_FOUND.getMessage(TEST_TEMP_BLOCK_ID2));
    mMetaManager.getTempBlockMeta(TEST_TEMP_BLOCK_ID2);
}

Example 8 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManagerTest method moveBlockMetaDeprecatedExceedCapacity.

/**
   * Tests that an exception is thrown in the
   * {@link BlockMetadataManager#moveBlockMeta(BlockMeta, BlockStoreLocation)} method when the
   * capacity is exceeded.
   */
@Test
public void moveBlockMetaDeprecatedExceedCapacity() throws Exception {
    StorageDir dir = mMetaManager.getTier("HDD").getDir(0);
    BlockMeta blockMeta = new BlockMeta(TEST_BLOCK_ID, 2000, dir);
    dir.addBlockMeta(blockMeta);
    mThrown.expect(WorkerOutOfSpaceException.class);
    mThrown.expectMessage("does not have enough space");
    mMetaManager.moveBlockMeta(blockMeta, new BlockStoreLocation("MEM", 0));
}

Example 9 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManagerViewTest method sameTierView.

/**
   * Tests that {@code BlockMetadataManagerView.getTierView(tierAlias)} returns the same
   * TierView as {@code new StorageTierView(mMetadataManager.getTier(tierAlias), this)}.
   */
@Test
public void sameTierView() {
    String tierAlias = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getTierAlias();
    StorageTierView tierView1 = mMetaManagerView.getTierView(tierAlias);
    // Do some operations on metadata
    StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getDir(TEST_DIR);
    BlockMeta blockMeta = new BlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir);
    try {
        dir.addBlockMeta(blockMeta);
    } catch (Exception e) {
        e.printStackTrace();
    }
    StorageTierView tierView2 = new StorageTierView(mMetaManager.getTier(tierAlias), mMetaManagerView);
    assertSameTierView(tierView1, tierView2);
}

Example 10 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class DefaultBlockWorker method moveBlock.

@Override
public void moveBlock(long sessionId, long blockId, String tierAlias) throws BlockDoesNotExistException, BlockAlreadyExistsException, InvalidWorkerStateException, WorkerOutOfSpaceException, IOException {
    // TODO(calvin): Move this logic into BlockStore#moveBlockInternal if possible
    // Because the move operation is expensive, we first check if the operation is necessary
    BlockStoreLocation dst = BlockStoreLocation.anyDirInTier(tierAlias);
    long lockId = mBlockStore.lockBlock(sessionId, blockId);
    try {
        BlockMeta meta = mBlockStore.getBlockMeta(sessionId, blockId, lockId);
        if (meta.getBlockLocation().belongsTo(dst)) {
            return;
        }
    } finally {
        mBlockStore.unlockBlock(lockId);
    }
    // Execute the block move if necessary
    mBlockStore.moveBlock(sessionId, blockId, dst);
}