Examples with NioDataBuffer alluxio.network.protocol.databuffer.NioDataBuffer used on opensource projects

Example 1 with NioDataBuffer

use of alluxio.network.protocol.databuffer.NioDataBuffer in project alluxio by Alluxio.

the class GrpcDataReader method readChunkInternal.

private DataBuffer readChunkInternal() throws IOException {
    Preconditions.checkState(!mClient.get().isShutdown(), "Data reader is closed while reading data chunks.");
    DataBuffer buffer = null;
    ReadResponse response = null;
    if (mStream instanceof GrpcDataMessageBlockingStream) {
        DataMessage<ReadResponse, DataBuffer> message = ((GrpcDataMessageBlockingStream<ReadRequest, ReadResponse>) mStream).receiveDataMessage(mDataTimeoutMs);
        if (message != null) {
            response = message.getMessage();
            buffer = message.getBuffer();
            if (buffer == null && response.hasChunk() && response.getChunk().hasData()) {
                // falls back to use chunk message for compatibility
                ByteBuffer byteBuffer = response.getChunk().getData().asReadOnlyByteBuffer();
                buffer = new NioDataBuffer(byteBuffer, byteBuffer.remaining());
            }
            Preconditions.checkState(buffer != null, "response should always contain chunk");
        }
    } else {
        response = mStream.receive(mDataTimeoutMs);
        if (response != null) {
            Preconditions.checkState(response.hasChunk() && response.getChunk().hasData(), "response should always contain chunk");
            ByteBuffer byteBuffer = response.getChunk().getData().asReadOnlyByteBuffer();
            buffer = new NioDataBuffer(byteBuffer, byteBuffer.remaining());
        }
    }
    if (response == null) {
        return null;
    }
    mPosToRead += buffer.readableBytes();
    try {
        mStream.send(mReadRequest.toBuilder().setOffsetReceived(mPosToRead).build());
    } catch (Exception e) {
        // nothing is done as the receipt is sent at best effort
        LOG.debug("Failed to send receipt of data to worker {} for request {}: {}.", mAddress, mReadRequest, e.getMessage());
    }
    Preconditions.checkState(mPosToRead - mReadRequest.getOffset() <= mReadRequest.getLength());
    return buffer;
}

Example 2 with NioDataBuffer

use of alluxio.network.protocol.databuffer.NioDataBuffer in project alluxio by Alluxio.

the class LocalFileDataReader method readChunk.

@Override
public DataBuffer readChunk() throws IOException {
    if (mPos >= mEnd) {
        return null;
    }
    ByteBuffer buffer = mReader.read(mPos, Math.min(mChunkSize, mEnd - mPos));
    DataBuffer dataBuffer = new NioDataBuffer(buffer, buffer.remaining());
    mPos += dataBuffer.getLength();
    MetricsSystem.counter(MetricKey.CLIENT_BYTES_READ_LOCAL.getName()).inc(dataBuffer.getLength());
    MetricsSystem.meter(MetricKey.CLIENT_BYTES_READ_LOCAL_THROUGHPUT.getName()).mark(dataBuffer.getLength());
    return dataBuffer;
}

Example 3 with NioDataBuffer

use of alluxio.network.protocol.databuffer.NioDataBuffer in project alluxio by Alluxio.

the class BufferCachingGrpcDataReader method readChunk.

/**
 * Reads a chunk of data.
 *
 * @return a chunk of data
 */
@Nullable
@VisibleForTesting
protected DataBuffer readChunk() throws IOException {
    Preconditions.checkState(!mClient.get().isShutdown(), "Data reader is closed while reading data chunks.");
    DataBuffer buffer = null;
    ReadResponse response = null;
    response = mStream.receive(mDataTimeoutMs);
    if (response == null) {
        return null;
    }
    Preconditions.checkState(response.hasChunk() && response.getChunk().hasData(), "response should always contain chunk");
    ByteBuffer byteBuffer = response.getChunk().getData().asReadOnlyByteBuffer();
    buffer = new NioDataBuffer(byteBuffer, byteBuffer.remaining());
    mPosToRead += buffer.readableBytes();
    try {
        mStream.send(mReadRequest.toBuilder().setOffsetReceived(mPosToRead).build());
    } catch (Exception e) {
        // nothing is done as the receipt is sent at best effort
        LOG.debug("Failed to send receipt of data to worker {} for request {}", mAddress, mReadRequest, e);
    }
    Preconditions.checkState(mPosToRead - mReadRequest.getOffset() <= mReadRequest.getLength());
    return buffer;
}

Example 4 with NioDataBuffer

use of alluxio.network.protocol.databuffer.NioDataBuffer in project alluxio by Alluxio.

the class BlockWorkerDataReader method readChunk.

@Override
public DataBuffer readChunk() throws IOException {
    if (mPos >= mEnd) {
        return null;
    }
    ByteBuffer buffer = mReader.read(mPos, Math.min(mChunkSize, mEnd - mPos));
    DataBuffer dataBuffer = new NioDataBuffer(buffer, buffer.remaining());
    mPos += dataBuffer.getLength();
    MetricsSystem.counter(MetricKey.WORKER_BYTES_READ_DIRECT.getName()).inc(dataBuffer.getLength());
    MetricsSystem.meter(MetricKey.WORKER_BYTES_READ_DIRECT_THROUGHPUT.getName()).mark(dataBuffer.getLength());
    return dataBuffer;
}

Example 5 with NioDataBuffer

use of alluxio.network.protocol.databuffer.NioDataBuffer in project alluxio by Alluxio.

the class TestBufferCachingGrpcDataReader method readChunk.

@Override
protected DataBuffer readChunk() {
    if (mTotalChunkNum == -1) {
        mTotalChunkNum = getTotalChunkNum();
    }
    if (mReadChunkNum >= mTotalChunkNum) {
        // finished reading
        return null;
    }
    ByteBuffer byteBuffer = null;
    if ((mReadChunkNum + 1) * mChunkSize <= mBlockSize) {
        // full block read
        byteBuffer = BufferUtils.getIncreasingByteBuffer(mReadChunkNum * mChunkSize, mChunkSize);
    } else {
        // partial block read for the last chunk
        byteBuffer = BufferUtils.getIncreasingByteBuffer(mReadChunkNum * mChunkSize, mBlockSize - mReadChunkNum * mChunkSize);
    }
    DataBuffer buffer = new NioDataBuffer(byteBuffer, byteBuffer.remaining());
    mPosToRead += buffer.readableBytes();
    mReadChunkNum++;
    return buffer;
}