Examples with StorageDir alluxio.worker.block.meta.StorageDir used on opensource projects

Example 21 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class LRFUAnnotatorTest method testLRFU.

@Test
public void testLRFU() throws Exception {
    Random rand = new Random();
    List<Long> expectedList = new ArrayList<>();
    for (long i = 0; i < 100; i++) {
        StorageDir pickedDir = getDir(rand.nextInt(2), rand.nextInt(2));
        createBlock(i, pickedDir);
        for (int j = 0; j < i * 2 + 1; j++) {
            accessBlock(i);
        }
        expectedList.add(i);
    }
    validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), expectedList.iterator());
    Collections.reverse(expectedList);
    validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.REVERSE), expectedList.iterator());
}

Example 22 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class AlignTaskTest method testTierAlignment.

@Test
public void testTierAlignment() throws Exception {
    Random rnd = new Random();
    StorageDir[] dirArray = new StorageDir[] { mTestDir1, mTestDir2, mTestDir3, mTestDir4 };
    // Start simulating random load on worker.
    startSimulateLoad();
    // Fill each directory.
    long sessionIdCounter = 1000;
    long blockIdCounter = 1000;
    for (StorageDir dir : dirArray) {
        while (dir.getAvailableBytes() > 0) {
            TieredBlockStoreTestUtils.cache(sessionIdCounter++, blockIdCounter++, BLOCK_SIZE, mBlockStore, dir.toBlockStoreLocation(), false);
        }
    }
    // Access blocks randomly.
    for (int i = 0; i < 100; i++) {
        StorageDir dirToAccess = dirArray[rnd.nextInt(dirArray.length)];
        List<Long> blockIdList = dirToAccess.getBlockIds();
        if (!blockIdList.isEmpty()) {
            mBlockStore.accessBlock(sessionIdCounter++, blockIdList.get(rnd.nextInt(blockIdList.size())));
        }
    }
    // Validate tiers are not aligned. (It's not guaranteed but using LRU helps.)
    Assert.assertTrue(!mBlockIterator.aligned(BlockStoreLocation.anyDirInTier(FIRST_TIER_ALIAS), BlockStoreLocation.anyDirInTier(SECOND_TIER_ALIAS), BlockOrder.NATURAL, (b) -> false));
    // Stop the load for align task to continue.
    stopSimulateLoad();
    CommonUtils.waitFor("Tiers to be aligned by a background task.", () -> mBlockIterator.aligned(BlockStoreLocation.anyDirInTier(FIRST_TIER_ALIAS), BlockStoreLocation.anyDirInTier(SECOND_TIER_ALIAS), BlockOrder.NATURAL, (b) -> false), WaitForOptions.defaults().setTimeoutMs(60000));
}

Example 23 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class EmulatingBlockIteratorTest method testLRUEmulation.

@Test
public void testLRUEmulation() throws Exception {
    // Block size that is common denominator to each dir capacity.
    final long blockSize = 1000;
    // Fill all directories and calculate expected block order for each of them as per LRU.
    Map<StorageDir, List<Long>> expectedListPerDir = new HashMap<>();
    long blockId = 0;
    for (StorageTier tier : mMetaManager.getTiers()) {
        for (StorageDir dir : tier.getStorageDirs()) {
            expectedListPerDir.put(dir, new LinkedList<>());
            while (dir.getAvailableBytes() > 0) {
                createBlock(++blockId, blockSize, dir);
                accessBlock(blockId);
                expectedListPerDir.get(dir).add(blockId);
            }
        }
    }
    // That's also why emulation can't be tested reliably with 'Reverse' iteration order.
    for (StorageDir dir : expectedListPerDir.keySet()) {
        validateIteratorHeader(mBlockIterator.getIterator(dir.toBlockStoreLocation(), BlockOrder.NATURAL), expectedListPerDir.get(dir).iterator());
    }
}

Example 24 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class LRUAnnotatorTest method testLRU.

@Test
public void testLRU() throws Exception {
    Random rand = new Random();
    List<Long> expectedList = new ArrayList<>();
    for (long i = 0; i < 100; i++) {
        StorageDir pickedDir = getDir(rand.nextInt(2), rand.nextInt(2));
        createBlock(i, pickedDir);
        accessBlock(i);
        expectedList.add(i);
    }
    validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.NATURAL), expectedList.iterator());
    Collections.reverse(expectedList);
    validateIterator(mBlockIterator.getIterator(BlockStoreLocation.anyTier(), BlockOrder.REVERSE), expectedList.iterator());
}

Example 25 with StorageDir

use of alluxio.worker.block.meta.StorageDir in project alluxio by Alluxio.

the class BlockMetadataManagerTest method moveBlockMetaDiffDir.

/**
 * Tests that an exception is thrown in the
 * {@link BlockMetadataManager#moveBlockMeta(BlockMeta, TempBlockMeta)} method when trying to move
 * a block to a not committed block meta.
 */
@Test
public void moveBlockMetaDiffDir() throws Exception {
    // create and add two temp block metas with different dirs in the same HDD tier
    StorageDir dir1 = mMetaManager.getTier(Constants.MEDIUM_HDD).getDir(0);
    StorageDir dir2 = mMetaManager.getTier(Constants.MEDIUM_HDD).getDir(1);
    TempBlockMeta tempBlockMeta1 = new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID, TEST_BLOCK_SIZE, dir1);
    TempBlockMeta tempBlockMeta2 = new DefaultTempBlockMeta(TEST_SESSION_ID, TEST_TEMP_BLOCK_ID2, TEST_BLOCK_SIZE, dir2);
    mMetaManager.addTempBlockMeta(tempBlockMeta1);
    mMetaManager.addTempBlockMeta(tempBlockMeta2);
    // commit the first temp block meta
    mMetaManager.commitTempBlockMeta(tempBlockMeta1);
    BlockMeta blockMeta = mMetaManager.getBlockMeta(TEST_TEMP_BLOCK_ID);
    mMetaManager.moveBlockMeta(blockMeta, tempBlockMeta2);
    // make sure that the dst tempBlockMeta has been removed from the dir2
    mThrown.expect(BlockDoesNotExistException.class);
    mThrown.expectMessage(ExceptionMessage.TEMP_BLOCK_META_NOT_FOUND.getMessage(TEST_TEMP_BLOCK_ID2));
    mMetaManager.getTempBlockMeta(TEST_TEMP_BLOCK_ID2);
}