Examples with DataBuffer alluxio.network.protocol.databuffer.DataBuffer used on opensource projects

Example 6 with DataBuffer

use of alluxio.network.protocol.databuffer.DataBuffer in project alluxio by Alluxio.

the class BlockDataServerHandler method handleBlockWriteRequest.

/**
   * Handles a {@link RPCBlockWriteRequest} by writing the data through a {@link BlockWriter}
   * provided by the block worker. This method takes care of requesting space and creating the
   * block if necessary.
   *
   * @param ctx The context of this request which handles the result of this operation
   * @param req The initiating {@link RPCBlockWriteRequest}
   * @throws IOException if an I/O exception occurs when writing the data
   */
// TODO(hy): This write request handler is very simple in order to be stateless. Therefore, the
// block file is opened and closed for every request. If this is too slow, then this handler
// should be optimized to keep state.
void handleBlockWriteRequest(final ChannelHandlerContext ctx, final RPCBlockWriteRequest req) throws IOException {
    final long sessionId = req.getSessionId();
    final long blockId = req.getBlockId();
    final long offset = req.getOffset();
    final long length = req.getLength();
    final DataBuffer data = req.getPayloadDataBuffer();
    BlockWriter writer = null;
    try {
        req.validate();
        ByteBuffer buffer = data.getReadOnlyByteBuffer();
        if (offset == 0) {
            // This is the first write to the block, so create the temp block file. The file will only
            // be created if the first write starts at offset 0. This allocates enough space for the
            // write.
            mWorker.createBlockRemote(sessionId, blockId, mStorageTierAssoc.getAlias(0), length);
        } else {
            // Allocate enough space in the existing temporary block for the write.
            mWorker.requestSpace(sessionId, blockId, length);
        }
        writer = mWorker.getTempBlockWriterRemote(sessionId, blockId);
        writer.append(buffer);
        Metrics.BYTES_WRITTEN_REMOTE.inc(data.getLength());
        RPCBlockWriteResponse resp = new RPCBlockWriteResponse(sessionId, blockId, offset, length, RPCResponse.Status.SUCCESS);
        ChannelFuture future = ctx.writeAndFlush(resp);
        future.addListener(new ClosableResourceChannelListener(writer));
    } catch (Exception e) {
        LOG.error("Error writing remote block : {}", e.getMessage(), e);
        RPCBlockWriteResponse resp = RPCBlockWriteResponse.createErrorResponse(req, RPCResponse.Status.WRITE_ERROR);
        ChannelFuture future = ctx.writeAndFlush(resp);
        future.addListener(ChannelFutureListener.CLOSE);
        if (writer != null) {
            writer.close();
        }
    }
}

Example 7 with DataBuffer

use of alluxio.network.protocol.databuffer.DataBuffer in project alluxio by Alluxio.

the class BlockDataServerHandler method handleBlockReadRequest.

/**
   * Handles a {@link RPCBlockReadRequest} by reading the data through a {@link BlockReader}
   * provided by the block worker. This method assumes the data is available in the local storage
   * of the worker and returns an error status if the data is not available.
   *
   * @param ctx The context of this request which handles the result of this operation
   * @param req The initiating {@link RPCBlockReadRequest}
   * @throws IOException if an I/O error occurs when reading the data requested
   */
void handleBlockReadRequest(final ChannelHandlerContext ctx, final RPCBlockReadRequest req) throws IOException {
    final long blockId = req.getBlockId();
    final long offset = req.getOffset();
    final long len = req.getLength();
    final long lockId = req.getLockId();
    final long sessionId = req.getSessionId();
    BlockReader reader = null;
    DataBuffer buffer;
    try {
        req.validate();
        reader = mWorker.readBlockRemote(sessionId, blockId, lockId);
        final long fileLength = reader.getLength();
        validateBounds(req, fileLength);
        final long readLength = returnLength(offset, len, fileLength);
        buffer = getDataBuffer(req, reader, readLength);
        Metrics.BYTES_READ_REMOTE.inc(buffer.getLength());
        RPCBlockReadResponse resp = new RPCBlockReadResponse(blockId, offset, readLength, buffer, RPCResponse.Status.SUCCESS);
        ChannelFuture future = ctx.writeAndFlush(resp);
        future.addListener(new ClosableResourceChannelListener(reader));
        future.addListener(new ReleasableResourceChannelListener(buffer));
        mWorker.accessBlock(sessionId, blockId);
        LOG.debug("Preparation for responding to remote block request for: {} done.", blockId);
    } catch (Exception e) {
        LOG.error("Exception reading block {}", blockId, e);
        RPCBlockReadResponse resp;
        if (e instanceof BlockDoesNotExistException) {
            resp = RPCBlockReadResponse.createErrorResponse(req, RPCResponse.Status.FILE_DNE);
        } else {
            resp = RPCBlockReadResponse.createErrorResponse(req, RPCResponse.Status.UFS_READ_FAILED);
        }
        ChannelFuture future = ctx.writeAndFlush(resp);
        future.addListener(ChannelFutureListener.CLOSE);
        if (reader != null) {
            reader.close();
        }
    }
}

Example 8 with DataBuffer

use of alluxio.network.protocol.databuffer.DataBuffer in project alluxio by Alluxio.

the class DataServerWriteHandler method channelRead.

@Override
public void channelRead(ChannelHandlerContext ctx, Object object) throws Exception {
    if (!acceptMessage(object)) {
        ctx.fireChannelRead(object);
        return;
    }
    RPCProtoMessage msg = (RPCProtoMessage) object;
    initializeRequest(msg);
    // Validate msg and return error if invalid. Init variables if necessary.
    String error = validateRequest(msg);
    if (!error.isEmpty()) {
        replyError(ctx.channel(), Protocol.Status.Code.INVALID_ARGUMENT, error, null);
        return;
    }
    mLock.lock();
    try {
        DataBuffer dataBuffer = msg.getPayloadDataBuffer();
        ByteBuf buf;
        if (dataBuffer == null) {
            buf = ctx.alloc().buffer(0, 0);
        } else {
            Preconditions.checkState(dataBuffer.getLength() > 0);
            assert dataBuffer.getNettyOutput() instanceof ByteBuf;
            buf = (ByteBuf) dataBuffer.getNettyOutput();
        }
        mPosToQueue += buf.readableBytes();
        mPackets.offer(buf);
        if (!mPacketWriterActive) {
            mPacketWriterExecutor.submit(new PacketWriter(ctx));
            mPacketWriterActive = true;
        }
        if (tooManyPacketsInFlight()) {
            ctx.channel().config().setAutoRead(false);
        }
    } finally {
        mLock.unlock();
    }
}

Example 9 with DataBuffer

use of alluxio.network.protocol.databuffer.DataBuffer in project alluxio by Alluxio.

the class UnderFileSystemDataServerHandler method handleFileReadRequest.

/**
   * Handles a {@link RPCFileReadRequest} by reading the data through an input stream provided by
   * the file worker. This method assumes the length to read is less than or equal to the unread
   * data in the file.
   *
   * @param ctx The context of this request which handles the result of this operation
   * @param req The initiating {@link RPCFileReadRequest}
   * @throws IOException if an I/O error occurs when interacting with the UFS
   */
public void handleFileReadRequest(ChannelHandlerContext ctx, RPCFileReadRequest req) throws IOException {
    req.validate();
    long ufsFileId = req.getTempUfsFileId();
    long offset = req.getOffset();
    long length = req.getLength();
    byte[] data = new byte[(int) length];
    try {
        InputStream in = mWorker.getUfsInputStream(ufsFileId, offset);
        int bytesRead = 0;
        if (in != null) {
            // if we have not reached the end of the file
            while (bytesRead < length) {
                // TODO(peis): Fix this. It is not recommended to do heavy blocking IO operation
                // in Netty's IO event group. We should do it in another threadpool.
                int read = in.read(data, bytesRead, (int) length - bytesRead);
                if (read == -1) {
                    break;
                }
                bytesRead += read;
            }
        }
        DataBuffer buf = bytesRead != 0 ? new DataByteBuffer(ByteBuffer.wrap(data, 0, bytesRead), bytesRead) : null;
        RPCFileReadResponse resp = new RPCFileReadResponse(ufsFileId, offset, bytesRead, buf, RPCResponse.Status.SUCCESS);
        ctx.writeAndFlush(resp);
    } catch (Exception e) {
        // TODO(peis): Fix this. The exception here should never be caused netty related issue.
        LOG.error("Failed to read ufs file, may have been closed due to a client timeout.", e);
        RPCFileReadResponse resp = RPCFileReadResponse.createErrorResponse(req, RPCResponse.Status.UFS_READ_FAILED);
        ChannelFuture future = ctx.writeAndFlush(resp);
        future.addListener(ChannelFutureListener.CLOSE);
    }
}

Example 10 with DataBuffer

use of alluxio.network.protocol.databuffer.DataBuffer in project alluxio by Alluxio.

the class LocalFilePacketReader method readPacket.

@Override
public DataBuffer readPacket() throws IOException {
    if (mPos >= mEnd) {
        return null;
    }
    ByteBuffer buffer = mReader.read(mPos, Math.min(LOCAL_READ_PACKET_SIZE, mEnd - mPos));
    DataBuffer dataBuffer = new DataByteBuffer(buffer, buffer.remaining());
    mPos += dataBuffer.getLength();
    return dataBuffer;
}