Examples with BlockReader alluxio.worker.block.io.BlockReader used on opensource projects

Example 6 with BlockReader

use of alluxio.worker.block.io.BlockReader in project alluxio by Alluxio.

the class UnderFileSystemBlockStore method getBlockReader.

/**
   * Creates a block reader that reads from UFS and optionally caches the block to the Alluxio
   * block store.
   *
   * @param sessionId the client session ID that requested this read
   * @param blockId the ID of the block to read
   * @param offset the read offset within the block (NOT the file)
   * @param noCache if set, do not try to cache the block in the Alluxio worker
   * @return the block reader instance
   * @throws BlockDoesNotExistException if the UFS block does not exist in the
   * {@link UnderFileSystemBlockStore}
   * @throws IOException if any I/O errors occur
   */
public BlockReader getBlockReader(final long sessionId, long blockId, long offset, boolean noCache) throws BlockDoesNotExistException, IOException {
    final BlockInfo blockInfo;
    mLock.lock();
    try {
        blockInfo = getBlockInfo(sessionId, blockId);
        BlockReader blockReader = blockInfo.getBlockReader();
        if (blockReader != null) {
            return blockReader;
        }
    } finally {
        mLock.unlock();
    }
    BlockReader reader = UnderFileSystemBlockReader.create(blockInfo.getMeta(), offset, noCache, mLocalBlockStore);
    blockInfo.setBlockReader(reader);
    return reader;
}

Example 7 with BlockReader

use of alluxio.worker.block.io.BlockReader in project alluxio by Alluxio.

the class KeyValueWorkerClientServiceHandler method getReader.

private ByteBufferKeyValuePartitionReader getReader(long sessionId, long lockId, long blockId) throws InvalidWorkerStateException, BlockDoesNotExistException, IOException {
    BlockReader blockReader = mBlockWorker.readBlockRemote(sessionId, blockId, lockId);
    ByteBuffer fileBuffer = blockReader.read(0, blockReader.getLength());
    ByteBufferKeyValuePartitionReader reader = new ByteBufferKeyValuePartitionReader(fileBuffer);
    // TODO(binfan): clean fileBuffer which is a direct byte buffer
    blockReader.close();
    return reader;
}

Example 8 with BlockReader

use of alluxio.worker.block.io.BlockReader in project alluxio by Alluxio.

the class FileDataManagerTest method persistFileRateLimiting.

/**
   * Tests the rate limiting functionality for asynchronous persistence.
   */
@Test
public void persistFileRateLimiting() throws Exception {
    Configuration.set(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT_ENABLED, "true");
    Configuration.set(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT, "100");
    mMockRateLimiter = new MockRateLimiter(Configuration.getBytes(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT));
    mManager = new FileDataManager(mBlockWorker, mMockRateLimiter.getGuavaRateLimiter());
    long fileId = 1;
    List<Long> blockIds = Lists.newArrayList(1L, 2L, 3L);
    FileInfo fileInfo = new FileInfo();
    fileInfo.setPath("test");
    Mockito.when(mBlockWorker.getFileInfo(fileId)).thenReturn(fileInfo);
    BlockReader reader = Mockito.mock(BlockReader.class);
    for (long blockId : blockIds) {
        Mockito.when(mBlockWorker.lockBlock(Sessions.CHECKPOINT_SESSION_ID, blockId)).thenReturn(blockId);
        Mockito.when(mBlockWorker.readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, blockId)).thenReturn(reader);
        BlockMeta mockedBlockMeta = PowerMockito.mock(BlockMeta.class);
        Mockito.when(mockedBlockMeta.getBlockSize()).thenReturn(100L);
        Mockito.when(mBlockWorker.getBlockMeta(Sessions.CHECKPOINT_SESSION_ID, blockId, blockId)).thenReturn(mockedBlockMeta);
    }
    String ufsRoot = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
    Mockito.when(mUfs.isDirectory(ufsRoot)).thenReturn(true);
    OutputStream outputStream = Mockito.mock(OutputStream.class);
    // mock BufferUtils
    PowerMockito.mockStatic(BufferUtils.class);
    String dstPath = PathUtils.concatPath(ufsRoot, fileInfo.getPath());
    fileInfo.setUfsPath(dstPath);
    Mockito.when(mUfs.create(dstPath)).thenReturn(outputStream);
    Mockito.when(mUfs.create(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(outputStream);
    Mockito.when(mMockFileSystem.getStatus(Mockito.any(AlluxioURI.class))).thenReturn(new URIStatus(fileInfo));
    mManager.lockBlocks(fileId, blockIds);
    mManager.persistFile(fileId, blockIds);
    List<String> expectedEvents = Lists.newArrayList("R0.00", "R1.00", "R1.00");
    assertEquals(expectedEvents, mMockRateLimiter.readEventsAndClear());
    // Simulate waiting for 1 second.
    mMockRateLimiter.sleepMillis(1000);
    mManager.lockBlocks(fileId, blockIds);
    mManager.persistFile(fileId, blockIds);
    // The first write will go through immediately without throttling.
    expectedEvents = Lists.newArrayList("U1.00", "R0.00", "R1.00", "R1.00");
    assertEquals(expectedEvents, mMockRateLimiter.readEventsAndClear());
    // Repeat persistence without sleeping.
    mManager.lockBlocks(fileId, blockIds);
    mManager.persistFile(fileId, blockIds);
    expectedEvents = Lists.newArrayList("R1.00", "R1.00", "R1.00");
    assertEquals(expectedEvents, mMockRateLimiter.readEventsAndClear());
}

Example 9 with BlockReader

use of alluxio.worker.block.io.BlockReader in project alluxio by Alluxio.

the class FileDataManagerTest method writeFileWithBlocks.

private void writeFileWithBlocks(long fileId, List<Long> blockIds) throws Exception {
    FileInfo fileInfo = new FileInfo();
    fileInfo.setPath("test");
    Mockito.when(mBlockWorker.getFileInfo(fileId)).thenReturn(fileInfo);
    BlockReader reader = Mockito.mock(BlockReader.class);
    for (long blockId : blockIds) {
        Mockito.when(mBlockWorker.lockBlock(Sessions.CHECKPOINT_SESSION_ID, blockId)).thenReturn(blockId);
        Mockito.when(mBlockWorker.readBlockRemote(Sessions.CHECKPOINT_SESSION_ID, blockId, blockId)).thenReturn(reader);
    }
    String ufsRoot = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
    Mockito.when(mUfs.isDirectory(ufsRoot)).thenReturn(true);
    OutputStream outputStream = Mockito.mock(OutputStream.class);
    // mock BufferUtils
    PowerMockito.mockStatic(BufferUtils.class);
    String dstPath = PathUtils.concatPath(ufsRoot, fileInfo.getPath());
    fileInfo.setUfsPath(dstPath);
    Mockito.when(mUfs.create(dstPath)).thenReturn(outputStream);
    Mockito.when(mUfs.create(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(outputStream);
    Mockito.when(mMockFileSystem.getStatus(Mockito.any(AlluxioURI.class))).thenReturn(new URIStatus(fileInfo));
    mManager.lockBlocks(fileId, blockIds);
    mManager.persistFile(fileId, blockIds);
}

Example 10 with BlockReader

use of alluxio.worker.block.io.BlockReader in project alluxio by Alluxio.

the class DataServerBlockReadHandlerTest method transferType.

/**
   * Tests the {@link FileTransferType#TRANSFER} type.
   */
@Test
public void transferType() throws Exception {
    mChannel = new EmbeddedChannel(new DataServerBlockReadHandler(NettyExecutors.BLOCK_READER_EXECUTOR, mBlockWorker, FileTransferType.TRANSFER));
    long fileSize = PACKET_SIZE * 2;
    long checksumExpected = populateInputFile(fileSize, 0, fileSize - 1);
    BlockReader blockReader = Mockito.spy(mBlockReader);
    // Do not call close here so that we can check result. It will be closed explicitly.
    Mockito.doNothing().when(blockReader).close();
    Mockito.when(mBlockWorker.readBlockRemote(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(blockReader);
    mChannel.writeInbound(buildReadRequest(0, fileSize));
    checkAllReadResponses(mChannel, checksumExpected);
    mBlockReader.close();
}