Examples with ByteBuf org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf used on opensource projects

Example 1 with ByteBuf

use of org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf in project incubator-ratis by apache.

the class NettyDataStreamUtils method decodeDataStreamReplyHeader.

static DataStreamReplyHeader decodeDataStreamReplyHeader(ByteBuf buf) {
    if (DataStreamPacketHeader.getSizeOfHeaderLen() > buf.readableBytes()) {
        return null;
    }
    int headerBufLen = buf.readInt();
    if (headerBufLen > buf.readableBytes()) {
        buf.resetReaderIndex();
        return null;
    }
    try {
        ByteBuf headerBuf = buf.slice(buf.readerIndex(), headerBufLen);
        DataStreamReplyHeaderProto header = DataStreamReplyHeaderProto.parseFrom(headerBuf.nioBuffer());
        final DataStreamPacketHeaderProto h = header.getPacketHeader();
        if (header.getPacketHeader().getDataLength() + headerBufLen <= buf.readableBytes()) {
            buf.readerIndex(buf.readerIndex() + headerBufLen);
            return new DataStreamReplyHeader(ClientId.valueOf(h.getClientId()), h.getType(), h.getStreamId(), h.getStreamOffset(), h.getDataLength(), header.getBytesWritten(), header.getSuccess(), header.getCommitInfosList());
        } else {
            buf.resetReaderIndex();
            return null;
        }
    } catch (InvalidProtocolBufferException e) {
        LOG.error("Fail to decode reply header:", e);
        buf.resetReaderIndex();
        return null;
    }
}

Example 2 with ByteBuf

use of org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf in project incubator-ratis by apache.

the class DataStreamManagement method readImpl.

private void readImpl(DataStreamRequestByteBuf request, ChannelHandlerContext ctx, ByteBuf buf, CheckedBiFunction<RaftClientRequest, Set<RaftPeer>, Set<DataStreamOutputRpc>, IOException> getStreams) {
    boolean close = WriteOption.containsOption(request.getWriteOptions(), StandardWriteOption.CLOSE);
    ClientInvocationId key = ClientInvocationId.valueOf(request.getClientId(), request.getStreamId());
    final StreamInfo info;
    if (request.getType() == Type.STREAM_HEADER) {
        final MemoizedSupplier<StreamInfo> supplier = JavaUtils.memoize(() -> newStreamInfo(buf, getStreams));
        info = streams.computeIfAbsent(key, id -> supplier.get());
        if (!supplier.isInitialized()) {
            streams.remove(key);
            throw new IllegalStateException("Failed to create a new stream for " + request + " since a stream already exists Key: " + key + " StreamInfo:" + info);
        }
        getMetrics().onRequestCreate(RequestType.HEADER);
    } else if (close) {
        info = Optional.ofNullable(streams.remove(key)).orElseThrow(() -> new IllegalStateException("Failed to remove StreamInfo for " + request));
    } else {
        info = Optional.ofNullable(streams.get(key)).orElseThrow(() -> {
            streams.remove(key);
            return new IllegalStateException("Failed to get StreamInfo for " + request);
        });
    }
    final CompletableFuture<Long> localWrite;
    final List<CompletableFuture<DataStreamReply>> remoteWrites;
    if (request.getType() == Type.STREAM_HEADER) {
        localWrite = CompletableFuture.completedFuture(0L);
        remoteWrites = Collections.emptyList();
    } else if (request.getType() == Type.STREAM_DATA) {
        localWrite = info.getLocal().write(buf, request.getWriteOptions(), writeExecutor);
        remoteWrites = info.applyToRemotes(out -> out.write(request, requestExecutor));
    } else {
        throw new IllegalStateException(this + ": Unexpected type " + request.getType() + ", request=" + request);
    }
    composeAsync(info.getPrevious(), requestExecutor, n -> JavaUtils.allOf(remoteWrites).thenCombineAsync(localWrite, (v, bytesWritten) -> {
        if (request.getType() == Type.STREAM_HEADER || (request.getType() == Type.STREAM_DATA && !close)) {
            sendReply(remoteWrites, request, bytesWritten, info.getCommitInfos(), ctx);
        } else if (close) {
            if (info.isPrimary()) {
                // after all server close stream, primary server start transaction
                startTransaction(info, request, bytesWritten, ctx);
            } else {
                sendReply(remoteWrites, request, bytesWritten, info.getCommitInfos(), ctx);
            }
        } else {
            throw new IllegalStateException(this + ": Unexpected type " + request.getType() + ", request=" + request);
        }
        return null;
    }, requestExecutor)).whenComplete((v, exception) -> {
        try {
            if (exception != null) {
                streams.remove(key);
                replyDataStreamException(server, exception, info.getRequest(), request, ctx);
            }
        } finally {
            buf.release();
        }
    });
}

Example 3 with ByteBuf

use of org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf in project incubator-ratis by apache.

the class DataStreamManagement method read.

void read(DataStreamRequestByteBuf request, ChannelHandlerContext ctx, CheckedBiFunction<RaftClientRequest, Set<RaftPeer>, Set<DataStreamOutputRpc>, IOException> getStreams) {
    LOG.debug("{}: read {}", this, request);
    final ByteBuf buf = request.slice();
    try {
        readImpl(request, ctx, buf, getStreams);
    } catch (Throwable t) {
        buf.release();
        throw t;
    }
}

Example 4 with ByteBuf

use of org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf in project incubator-ratis by apache.

the class LoadGen method write.

long write(FileChannel in, long offset, FileStoreClient fileStoreClient, String path, List<CompletableFuture<Long>> futures) throws IOException {
    final int bufferSize = getBufferSizeInBytes();
    final ByteBuf buf = PooledByteBufAllocator.DEFAULT.heapBuffer(bufferSize);
    final int bytesRead = buf.writeBytes(in, bufferSize);
    if (bytesRead < 0) {
        throw new IllegalStateException("Failed to read " + bufferSize + " byte(s) from " + this + ". The channel has reached end-of-stream at " + offset);
    } else if (bytesRead > 0) {
        final CompletableFuture<Long> f = fileStoreClient.writeAsync(path, offset, offset + bytesRead == getFileSizeInBytes(), buf.nioBuffer(), sync == 1);
        f.thenRun(buf::release);
        futures.add(f);
    }
    return bytesRead;
}

Example 5 with ByteBuf

use of org.apache.ratis.thirdparty.io.netty.buffer.ByteBuf in project incubator-ratis by apache.

the class RequestDecoder method decode.

@Override
protected void decode(ChannelHandlerContext ctx, ByteBuf msg, List<Object> out) throws Exception {
    if (msg.readableBytes() >= 8) {
        int id = msg.readInt();
        int buflen = msg.readInt();
        if (msg.readableBytes() >= buflen) {
            RequestData req = new RequestData();
            req.setDataId(id);
            // System.out.printf("msg id and buflen %d and %d bytes\n", id, buflen, msg.readableBytes());
            try {
                ByteBuf bf = msg.slice(msg.readerIndex(), buflen);
                req.setBuff(bf.nioBuffer());
            } catch (Exception e) {
                System.out.println(e);
            }
            msg.readerIndex(msg.readerIndex() + buflen);
            msg.markReaderIndex();
            out.add(req);
        } else {
            msg.resetReaderIndex();
            return;
        }
    } else {
        return;
    }
}