Examples with ReadRequest alluxio.grpc.ReadRequest used on opensource projects

Example 1 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class BufferCachingGrpcDataReader method create.

/**
 * Creates an instance of {@link BufferCachingGrpcDataReader} for block reads.
 *
 * @param context the file system context
 * @param address the worker address
 * @param readRequest the read request
 * @return a new {@link BufferCachingGrpcDataReader}
 */
public static BufferCachingGrpcDataReader create(FileSystemContext context, WorkerNetAddress address, ReadRequest readRequest) throws IOException {
    AlluxioConfiguration alluxioConf = context.getClusterConf();
    int readerBufferSizeMessages = alluxioConf.getInt(PropertyKey.USER_STREAMING_READER_BUFFER_SIZE_MESSAGES);
    long dataTimeoutMs = alluxioConf.getMs(PropertyKey.USER_STREAMING_DATA_READ_TIMEOUT);
    CloseableResource<BlockWorkerClient> client = context.acquireBlockWorkerClient(address);
    String desc = "BufferCachingGrpcDataReader";
    if (LOG.isDebugEnabled()) {
        // More detailed description when debug logging is enabled
        desc = String.format("BufferCachingGrpcDataReader(request=%s,address=%s)", readRequest, address);
    }
    GrpcBlockingStream<ReadRequest, ReadResponse> stream = null;
    try {
        // Stream here cannot be GrpcDataMessagingBlockingStream
        // DataBuffer.getReadOnlyByteBuffer is used to clone a copy in SharedDataReader.readChunk.
        // getReadOnlyByteBuffer is not implemented in DataBuffer
        // returned from GrpcDataMessagingBlockingStream.
        stream = new GrpcBlockingStream<>(client.get()::readBlock, readerBufferSizeMessages, desc);
        stream.send(readRequest, dataTimeoutMs);
    } catch (Exception e) {
        if (stream != null) {
            stream.close();
        }
        client.close();
        throw e;
    }
    return new BufferCachingGrpcDataReader(address, client, dataTimeoutMs, readRequest, stream);
}

Example 2 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class BlockInStream method createRemoteBlockInStream.

/**
 * Creates a {@link BlockInStream} to read from a specific remote server. Should only be used
 * in cases where the data source and method of reading is known, ie. worker - worker
 * communication.
 *
 * @param context the file system context
 * @param blockId the block id
 * @param address the address of the gRPC data server
 * @param blockSource the source location of the block
 * @param blockSize the size of the block
 * @param ufsOptions the ufs read options
 * @return the {@link BlockInStream} created
 */
public static BlockInStream createRemoteBlockInStream(FileSystemContext context, long blockId, WorkerNetAddress address, BlockInStreamSource blockSource, long blockSize, Protocol.OpenUfsBlockOptions ufsOptions) {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES);
    ReadRequest readRequest = ReadRequest.newBuilder().setBlockId(blockId).setOpenUfsBlockOptions(ufsOptions).setChunkSize(chunkSize).buildPartial();
    DataReader.Factory factory = new GrpcDataReader.Factory(context, address, readRequest.toBuilder());
    return new BlockInStream(factory, conf, address, blockSource, blockId, blockSize);
}

Example 3 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class BufferCachingGrpcDataReaderTest method before.

@Before
public void before() throws Exception {
    WorkerNetAddress address = new WorkerNetAddress();
    BlockWorkerClient client = Mockito.mock(BlockWorkerClient.class);
    GrpcBlockingStream<ReadRequest, ReadResponse> unusedStream = new GrpcBlockingStream<>(client::readBlock, 5, "test message");
    ReadRequest readRequest = ReadRequest.newBuilder().setOffset(0).setLength(BLOCK_SIZE).setChunkSize(CHUNK_SIZE).setBlockId(1L).build();
    mDataReader = new TestBufferCachingGrpcDataReader(address, new NoopClosableResource<>(client), TIMEOUT, readRequest, unusedStream, CHUNK_SIZE, BLOCK_SIZE);
}

Example 4 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class GrpcDataReaderTest method validateReadRequestSent.

/**
 * Validates the read request sent.
 *
 * @param client the worker client
 * @param offset the offset
 * @param length the length
 */
private void validateReadRequestSent(final BlockWorkerClient client, long offset, long length, boolean closed, int chunkSize) throws TimeoutException, InterruptedException {
    ArgumentCaptor<ReadRequest> requestCaptor = ArgumentCaptor.forClass(ReadRequest.class);
    verify(mRequestObserver, atLeastOnce()).onNext(requestCaptor.capture());
    ArgumentCaptor<StreamObserver> captor = ArgumentCaptor.forClass(StreamObserver.class);
    verify(mClient).readBlock(captor.capture());
    List<ReadRequest> readRequests = requestCaptor.getAllValues();
    assertTrue(!readRequests.isEmpty());
    captor.getValue().onCompleted();
    long lastOffset = offset;
    for (int i = 0; i < readRequests.size(); i++) {
        ReadRequest readRequest = readRequests.get(i);
        if (i == 0) {
            assertTrue(readRequest != null);
            assertEquals(BLOCK_ID, readRequest.getBlockId());
            assertEquals(offset, readRequest.getOffset());
            assertEquals(length, readRequest.getLength());
            assertEquals(chunkSize, readRequest.getChunkSize());
        } else {
            assertTrue(readRequest.hasOffsetReceived());
            assertTrue(readRequest.getOffsetReceived() > lastOffset);
            assertTrue(readRequest.getOffsetReceived() <= length);
            lastOffset = readRequest.getOffsetReceived();
        }
    }
    verify(mRequestObserver, closed ? atLeastOnce() : never()).onCompleted();
}

Example 5 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class SharedGrpcDataReaderTest method before.

@Before
public void before() {
    WorkerNetAddress address = new WorkerNetAddress();
    BlockWorkerClient client = Mockito.mock(BlockWorkerClient.class);
    GrpcBlockingStream<ReadRequest, ReadResponse> unusedStream = new GrpcBlockingStream<>(client::readBlock, 5, "test message");
    mReadRequest = ReadRequest.newBuilder().setOffset(0).setLength(mBlockSize).setChunkSize(CHUNK_SIZE).setBlockId(BLOCK_ID).build();
    mBufferCachingDataReader = new TestBufferCachingGrpcDataReader(address, new NoopClosableResource<>(client), TIMEOUT, mReadRequest, unusedStream, CHUNK_SIZE, mBlockSize);
}