Examples with ReadRequest alluxio.grpc.ReadRequest used on opensource projects

Example 6 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class BufferCachingGrpcDataReader method create.

/**
 * Creates an instance of {@link BufferCachingGrpcDataReader} for block reads.
 *
 * @param context the file system context
 * @param address the worker address
 * @param readRequest the read request
 * @return a new {@link BufferCachingGrpcDataReader}
 */
public static BufferCachingGrpcDataReader create(FileSystemContext context, WorkerNetAddress address, ReadRequest readRequest) throws IOException {
    AlluxioConfiguration alluxioConf = context.getClusterConf();
    int readerBufferSizeMessages = alluxioConf.getInt(PropertyKey.USER_STREAMING_READER_BUFFER_SIZE_MESSAGES);
    long dataTimeoutMs = alluxioConf.getMs(PropertyKey.USER_STREAMING_DATA_READ_TIMEOUT);
    CloseableResource<BlockWorkerClient> client = context.acquireBlockWorkerClient(address);
    String desc = "BufferCachingGrpcDataReader";
    if (LOG.isDebugEnabled()) {
        // More detailed description when debug logging is enabled
        desc = String.format("BufferCachingGrpcDataReader(request=%s,address=%s)", readRequest, address);
    }
    GrpcBlockingStream<ReadRequest, ReadResponse> stream = null;
    try {
        // Stream here cannot be GrpcDataMessagingBlockingStream
        // DataBuffer.getReadOnlyByteBuffer is used to clone a copy in SharedDataReader.readChunk.
        // getReadOnlyByteBuffer is not implemented in DataBuffer
        // returned from GrpcDataMessagingBlockingStream.
        stream = new GrpcBlockingStream<>(client.get()::readBlock, readerBufferSizeMessages, desc);
        stream.send(readRequest, dataTimeoutMs);
    } catch (Exception e) {
        if (stream != null) {
            stream.close();
        }
        client.close();
        throw e;
    }
    return new BufferCachingGrpcDataReader(address, client, dataTimeoutMs, readRequest, stream);
}

Example 7 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class BlockInStream method createRemoteBlockInStream.

/**
 * Creates a {@link BlockInStream} to read from a specific remote server. Should only be used
 * in cases where the data source and method of reading is known, ie. worker - worker
 * communication.
 *
 * @param context the file system context
 * @param blockId the block id
 * @param address the address of the gRPC data server
 * @param blockSource the source location of the block
 * @param blockSize the size of the block
 * @param ufsOptions the ufs read options
 * @return the {@link BlockInStream} created
 */
public static BlockInStream createRemoteBlockInStream(FileSystemContext context, long blockId, WorkerNetAddress address, BlockInStreamSource blockSource, long blockSize, Protocol.OpenUfsBlockOptions ufsOptions) {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_STREAMING_READER_CHUNK_SIZE_BYTES);
    ReadRequest readRequest = ReadRequest.newBuilder().setBlockId(blockId).setOpenUfsBlockOptions(ufsOptions).setChunkSize(chunkSize).buildPartial();
    DataReader.Factory factory = new GrpcDataReader.Factory(context, address, readRequest.toBuilder());
    return new BlockInStream(factory, conf, address, blockSource, blockId, blockSize);
}

Example 8 with ReadRequest

use of alluxio.grpc.ReadRequest in project alluxio by Alluxio.

the class SharedGrpcDataReaderTest method twoThreadSequentialRead.

@Test(timeout = 1000 * 60)
public void twoThreadSequentialRead() throws Exception {
    int readerOneOffset = 10;
    int readerOneLen = CHUNK_SIZE * 6 + CHUNK_SIZE / 3;
    int readerTwoOffset = 80;
    int readerTwoLen = CHUNK_SIZE * 5 + CHUNK_SIZE / 2;
    ReadRequest readRequestOne = ReadRequest.newBuilder().setBlockId(BLOCK_ID).setChunkSize(CHUNK_SIZE).setOffset(readerOneOffset).setLength(readerOneLen).build();
    ReadRequest readRequestTwo = ReadRequest.newBuilder().setBlockId(BLOCK_ID).setChunkSize(CHUNK_SIZE).setOffset(readerTwoOffset).setLength(readerTwoLen).build();
    SharedGrpcDataReader sharedReaderOne = new SharedGrpcDataReader(readRequestOne, mBufferCachingDataReader);
    SharedGrpcDataReader sharedReaderTwo = new SharedGrpcDataReader(readRequestTwo, mBufferCachingDataReader);
    boolean readerOne = true;
    while (readerOneOffset != -1 && readerTwoOffset != -1) {
        if (readerOne) {
            readerOneOffset = validateRead(sharedReaderOne, readerOneOffset, getChunkNum(readerOneLen));
        } else {
            readerTwoOffset = validateRead(sharedReaderTwo, readerTwoOffset, getChunkNum(readerTwoLen));
        }
        readerOne = !readerOne;
    }
}