Examples with BlockMeta alluxio.worker.block.meta.BlockMeta used on opensource projects

Example 21 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class FileDataManagerTest method persistFileRateLimiting.

/**
   * Tests the rate limiting functionality for asynchronous persistence.
   */
@Test
public void persistFileRateLimiting() throws Exception {
    Configuration.set(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT_ENABLED, "true");
    Configuration.set(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT, "100");
    mMockRateLimiter = new MockRateLimiter(Configuration.getBytes(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT));
    mManager = new FileDataManager(mBlockWorker, mMockRateLimiter.getGuavaRateLimiter());
    long fileId = 1;
    List<Long> blockIds = Lists.newArrayList(1L, 2L, 3L);
    FileInfo fileInfo = new FileInfo();
    fileInfo.setPath("test");
    Mockito.when(mBlockWorker.getFileInfo(fileId)).thenReturn(fileInfo);
    BlockReader reader = Mockito.mock(BlockReader.class);
    for (long blockId : blockIds) {
        Mockito.when(mBlockWorker.lockBlock(Sessions.CHECKPOINT_SESSION_ID, blockId)).thenReturn(blockId);
        Mockito.when(mBlockWorker.readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, blockId)).thenReturn(reader);
        BlockMeta mockedBlockMeta = PowerMockito.mock(BlockMeta.class);
        Mockito.when(mockedBlockMeta.getBlockSize()).thenReturn(100L);
        Mockito.when(mBlockWorker.getBlockMeta(Sessions.CHECKPOINT_SESSION_ID, blockId, blockId)).thenReturn(mockedBlockMeta);
    }
    String ufsRoot = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
    Mockito.when(mUfs.isDirectory(ufsRoot)).thenReturn(true);
    OutputStream outputStream = Mockito.mock(OutputStream.class);
    // mock BufferUtils
    PowerMockito.mockStatic(BufferUtils.class);
    String dstPath = PathUtils.concatPath(ufsRoot, fileInfo.getPath());
    fileInfo.setUfsPath(dstPath);
    Mockito.when(mUfs.create(dstPath)).thenReturn(outputStream);
    Mockito.when(mUfs.create(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(outputStream);
    Mockito.when(mMockFileSystem.getStatus(Mockito.any(AlluxioURI.class))).thenReturn(new URIStatus(fileInfo));
    mManager.lockBlocks(fileId, blockIds);
    mManager.persistFile(fileId, blockIds);
    List<String> expectedEvents = Lists.newArrayList("R0.00", "R1.00", "R1.00");
    assertEquals(expectedEvents, mMockRateLimiter.readEventsAndClear());
    // Simulate waiting for 1 second.
    mMockRateLimiter.sleepMillis(1000);
    mManager.lockBlocks(fileId, blockIds);
    mManager.persistFile(fileId, blockIds);
    // The first write will go through immediately without throttling.
    expectedEvents = Lists.newArrayList("U1.00", "R0.00", "R1.00", "R1.00");
    assertEquals(expectedEvents, mMockRateLimiter.readEventsAndClear());
    // Repeat persistence without sleeping.
    mManager.lockBlocks(fileId, blockIds);
    mManager.persistFile(fileId, blockIds);
    expectedEvents = Lists.newArrayList("R1.00", "R1.00", "R1.00");
    assertEquals(expectedEvents, mMockRateLimiter.readEventsAndClear());
}

Example 22 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManagerViewTest method sameTierViewsBelow.

/**
   * Tests that {@link BlockMetadataManagerView#getTierViewsBelow(String)} returns the same
   * TierViews as constructing by {@link BlockMetadataManager#getTiersBelow(String)}.
   */
@Test
public void sameTierViewsBelow() {
    String tierAlias = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getTierAlias();
    List<StorageTierView> tierViews1 = mMetaManagerView.getTierViewsBelow(tierAlias);
    // Do some operations on metadata
    StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL + 1).getDir(TEST_DIR);
    BlockMeta blockMeta = new BlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir);
    try {
        dir.addBlockMeta(blockMeta);
    } catch (Exception e) {
        e.printStackTrace();
    }
    List<StorageTier> tiers2 = mMetaManager.getTiersBelow(tierAlias);
    Assert.assertEquals(tierViews1.size(), tiers2.size());
    for (int i = 0; i < tierViews1.size(); i++) {
        assertSameTierView(tierViews1.get(i), new StorageTierView(tiers2.get(i), mMetaManagerView));
    }
}

Example 23 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockMetadataManagerViewTest method getBlockMeta.

/**
   * Tests the {@link BlockMetadataManagerView#getBlockMeta(long)}  method.
   */
@Test
public void getBlockMeta() throws Exception {
    StorageDir dir = mMetaManager.getTiers().get(TEST_TIER_ORDINAL).getDir(TEST_DIR);
    // Add one block to test dir, expect block meta found
    BlockMeta blockMeta = new BlockMeta(TEST_BLOCK_ID, TEST_BLOCK_SIZE, dir);
    dir.addBlockMeta(blockMeta);
    Assert.assertEquals(blockMeta, mMetaManagerView.getBlockMeta(TEST_BLOCK_ID));
    Assert.assertTrue(mMetaManagerView.isBlockEvictable(TEST_BLOCK_ID));
    // Lock this block, expect null result
    Mockito.when(mMetaManagerView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(false);
    Mockito.when(mMetaManagerView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(true);
    Assert.assertNull(mMetaManagerView.getBlockMeta(TEST_BLOCK_ID));
    Assert.assertFalse(mMetaManagerView.isBlockEvictable(TEST_BLOCK_ID));
    // Pin this block, expect null result
    Mockito.when(mMetaManagerView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(true);
    Mockito.when(mMetaManagerView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(false);
    Assert.assertNull(mMetaManagerView.getBlockMeta(TEST_BLOCK_ID));
    Assert.assertFalse(mMetaManagerView.isBlockEvictable(TEST_BLOCK_ID));
    // No Pin or lock on this block, expect block meta found
    Mockito.when(mMetaManagerView.isBlockPinned(TEST_BLOCK_ID)).thenReturn(false);
    Mockito.when(mMetaManagerView.isBlockLocked(TEST_BLOCK_ID)).thenReturn(false);
    Assert.assertEquals(blockMeta, mMetaManagerView.getBlockMeta(TEST_BLOCK_ID));
    Assert.assertTrue(mMetaManagerView.isBlockEvictable(TEST_BLOCK_ID));
}

Example 24 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockWorkerTest method commitBlockOnRetry.

/**
   * Tests that commitBlock doesn't throw an exception when {@link BlockAlreadyExistsException} gets
   * thrown by the block store.
   */
@Test
public void commitBlockOnRetry() throws Exception {
    long blockId = mRandom.nextLong();
    long length = mRandom.nextLong();
    long lockId = mRandom.nextLong();
    long sessionId = mRandom.nextLong();
    long usedBytes = mRandom.nextLong();
    String tierAlias = "MEM";
    HashMap<String, Long> usedBytesOnTiers = new HashMap<>();
    usedBytesOnTiers.put(tierAlias, usedBytes);
    BlockMeta blockMeta = PowerMockito.mock(BlockMeta.class);
    BlockStoreLocation blockStoreLocation = PowerMockito.mock(BlockStoreLocation.class);
    BlockStoreMeta blockStoreMeta = PowerMockito.mock(BlockStoreMeta.class);
    when(mBlockStore.lockBlock(sessionId, blockId)).thenReturn(lockId);
    when(mBlockStore.getBlockMeta(sessionId, blockId, lockId)).thenReturn(blockMeta);
    when(mBlockStore.getBlockStoreMeta()).thenReturn(blockStoreMeta);
    when(blockMeta.getBlockLocation()).thenReturn(blockStoreLocation);
    when(blockStoreLocation.tierAlias()).thenReturn(tierAlias);
    when(blockMeta.getBlockSize()).thenReturn(length);
    when(blockStoreMeta.getUsedBytesOnTiers()).thenReturn(usedBytesOnTiers);
    doThrow(new BlockAlreadyExistsException("")).when(mBlockStore).commitBlock(sessionId, blockId);
    mBlockWorker.commitBlock(sessionId, blockId);
}

Example 25 with BlockMeta

use of alluxio.worker.block.meta.BlockMeta in project alluxio by Alluxio.

the class BlockWorkerTest method moveBlockNoop.

/**
   * Tests the {@link BlockWorker#moveBlock(long, long, String)} method no-ops if the block is
   * already at the destination location.
   */
@Test
public void moveBlockNoop() throws Exception {
    long blockId = mRandom.nextLong();
    long sessionId = mRandom.nextLong();
    String tierAlias = "MEM";
    BlockStoreLocation location = BlockStoreLocation.anyDirInTier(tierAlias);
    BlockStoreLocation existingLocation = Mockito.mock(BlockStoreLocation.class);
    when(existingLocation.belongsTo(location)).thenReturn(true);
    BlockMeta meta = Mockito.mock(BlockMeta.class);
    when(meta.getBlockLocation()).thenReturn(existingLocation);
    when(mBlockStore.getBlockMeta(Mockito.eq(sessionId), Mockito.eq(blockId), Mockito.anyLong())).thenReturn(meta);
    mBlockWorker.moveBlock(sessionId, blockId, tierAlias);
    verify(mBlockStore, Mockito.times(0)).moveBlock(sessionId, blockId, location);
}