Examples with ReadResponse alluxio.grpc.ReadResponse used on opensource projects

Example 1 with ReadResponse

use of alluxio.grpc.ReadResponse in project alluxio by Alluxio.

the class GrpcDataReaderTest method setReadResponses.

/**
 * Sets read responses which will be returned when readBlock() is invoked.
 *
 * @param client the worker client
 * @param length the length
 * @param start the start position to calculate the checksum
 * @param end the end position to calculate the checksum
 * @return the checksum
 */
private long setReadResponses(final BlockWorkerClient client, final long length, final long start, final long end) {
    long checksum = 0;
    long pos = 0;
    long remaining = length;
    ArgumentCaptor<StreamObserver> captor = ArgumentCaptor.forClass(StreamObserver.class);
    verify(mClient).readBlock(captor.capture());
    StreamObserver<ReadResponse> responseObserver = captor.getValue();
    List<ReadResponse> responses = new ArrayList<>();
    while (remaining > 0) {
        int bytesToSend = (int) Math.min(remaining, CHUNK_SIZE);
        byte[] data = new byte[bytesToSend];
        RANDOM.nextBytes(data);
        responses.add(ReadResponse.newBuilder().setChunk(Chunk.newBuilder().setData(ByteString.copyFrom(data))).build());
        remaining -= bytesToSend;
        for (int i = 0; i < data.length; i++) {
            if (pos >= start && pos <= end) {
                checksum += BufferUtils.byteToInt(data[i]);
            }
            pos++;
        }
    }
    EXECUTOR.submit(() -> {
        for (ReadResponse response : responses) {
            responseObserver.onNext(response);
        }
        responseObserver.onCompleted();
    });
    return checksum;
}

Example 2 with ReadResponse

use of alluxio.grpc.ReadResponse in project alluxio by Alluxio.

the class SharedGrpcDataReaderTest method before.

@Before
public void before() {
    WorkerNetAddress address = new WorkerNetAddress();
    BlockWorkerClient client = Mockito.mock(BlockWorkerClient.class);
    GrpcBlockingStream<ReadRequest, ReadResponse> unusedStream = new GrpcBlockingStream<>(client::readBlock, 5, "test message");
    mReadRequest = ReadRequest.newBuilder().setOffset(0).setLength(mBlockSize).setChunkSize(CHUNK_SIZE).setBlockId(BLOCK_ID).build();
    mBufferCachingDataReader = new TestBufferCachingGrpcDataReader(address, new NoopClosableResource<>(client), TIMEOUT, mReadRequest, unusedStream, CHUNK_SIZE, mBlockSize);
}

Example 3 with ReadResponse

use of alluxio.grpc.ReadResponse in project alluxio by Alluxio.

the class BufferCachingGrpcDataReaderTest method before.

@Before
public void before() throws Exception {
    WorkerNetAddress address = new WorkerNetAddress();
    BlockWorkerClient client = Mockito.mock(BlockWorkerClient.class);
    GrpcBlockingStream<ReadRequest, ReadResponse> unusedStream = new GrpcBlockingStream<>(client::readBlock, 5, "test message");
    ReadRequest readRequest = ReadRequest.newBuilder().setOffset(0).setLength(BLOCK_SIZE).setChunkSize(CHUNK_SIZE).setBlockId(1L).build();
    mDataReader = new TestBufferCachingGrpcDataReader(address, new NoopClosableResource<>(client), TIMEOUT, readRequest, unusedStream, CHUNK_SIZE, BLOCK_SIZE);
}

Example 4 with ReadResponse

use of alluxio.grpc.ReadResponse in project alluxio by Alluxio.

the class GrpcDataReader method readChunkInternal.

private DataBuffer readChunkInternal() throws IOException {
    Preconditions.checkState(!mClient.get().isShutdown(), "Data reader is closed while reading data chunks.");
    DataBuffer buffer = null;
    ReadResponse response = null;
    if (mStream instanceof GrpcDataMessageBlockingStream) {
        DataMessage<ReadResponse, DataBuffer> message = ((GrpcDataMessageBlockingStream<ReadRequest, ReadResponse>) mStream).receiveDataMessage(mDataTimeoutMs);
        if (message != null) {
            response = message.getMessage();
            buffer = message.getBuffer();
            if (buffer == null && response.hasChunk() && response.getChunk().hasData()) {
                // falls back to use chunk message for compatibility
                ByteBuffer byteBuffer = response.getChunk().getData().asReadOnlyByteBuffer();
                buffer = new NioDataBuffer(byteBuffer, byteBuffer.remaining());
            }
            Preconditions.checkState(buffer != null, "response should always contain chunk");
        }
    } else {
        response = mStream.receive(mDataTimeoutMs);
        if (response != null) {
            Preconditions.checkState(response.hasChunk() && response.getChunk().hasData(), "response should always contain chunk");
            ByteBuffer byteBuffer = response.getChunk().getData().asReadOnlyByteBuffer();
            buffer = new NioDataBuffer(byteBuffer, byteBuffer.remaining());
        }
    }
    if (response == null) {
        return null;
    }
    mPosToRead += buffer.readableBytes();
    try {
        mStream.send(mReadRequest.toBuilder().setOffsetReceived(mPosToRead).build());
    } catch (Exception e) {
        // nothing is done as the receipt is sent at best effort
        LOG.debug("Failed to send receipt of data to worker {} for request {}: {}.", mAddress, mReadRequest, e.getMessage());
    }
    Preconditions.checkState(mPosToRead - mReadRequest.getOffset() <= mReadRequest.getLength());
    return buffer;
}

Example 5 with ReadResponse

use of alluxio.grpc.ReadResponse in project alluxio by Alluxio.

the class BufferCachingGrpcDataReader method create.

/**
 * Creates an instance of {@link BufferCachingGrpcDataReader} for block reads.
 *
 * @param context the file system context
 * @param address the worker address
 * @param readRequest the read request
 * @return a new {@link BufferCachingGrpcDataReader}
 */
public static BufferCachingGrpcDataReader create(FileSystemContext context, WorkerNetAddress address, ReadRequest readRequest) throws IOException {
    AlluxioConfiguration alluxioConf = context.getClusterConf();
    int readerBufferSizeMessages = alluxioConf.getInt(PropertyKey.USER_STREAMING_READER_BUFFER_SIZE_MESSAGES);
    long dataTimeoutMs = alluxioConf.getMs(PropertyKey.USER_STREAMING_DATA_READ_TIMEOUT);
    CloseableResource<BlockWorkerClient> client = context.acquireBlockWorkerClient(address);
    String desc = "BufferCachingGrpcDataReader";
    if (LOG.isDebugEnabled()) {
        // More detailed description when debug logging is enabled
        desc = String.format("BufferCachingGrpcDataReader(request=%s,address=%s)", readRequest, address);
    }
    GrpcBlockingStream<ReadRequest, ReadResponse> stream = null;
    try {
        // Stream here cannot be GrpcDataMessagingBlockingStream
        // DataBuffer.getReadOnlyByteBuffer is used to clone a copy in SharedDataReader.readChunk.
        // getReadOnlyByteBuffer is not implemented in DataBuffer
        // returned from GrpcDataMessagingBlockingStream.
        stream = new GrpcBlockingStream<>(client.get()::readBlock, readerBufferSizeMessages, desc);
        stream.send(readRequest, dataTimeoutMs);
    } catch (Exception e) {
        if (stream != null) {
            stream.close();
        }
        client.close();
        throw e;
    }
    return new BufferCachingGrpcDataReader(address, client, dataTimeoutMs, readRequest, stream);
}