Examples with DataStreamReply org.apache.ratis.protocol.DataStreamReply used on opensource projects

Example 6 with DataStreamReply

use of org.apache.ratis.protocol.DataStreamReply in project incubator-ratis by apache.

the class DataStream method streamWrite.

private Map<String, CompletableFuture<List<CompletableFuture<DataStreamReply>>>> streamWrite(List<String> paths, List<FileStoreClient> clients, RoutingTable routingTable, ExecutorService executor) {
    Map<String, CompletableFuture<List<CompletableFuture<DataStreamReply>>>> fileMap = new HashMap<>();
    int clientIndex = 0;
    for (String path : paths) {
        final CompletableFuture<List<CompletableFuture<DataStreamReply>>> future = new CompletableFuture<>();
        final FileStoreClient client = clients.get(clientIndex % clients.size());
        clientIndex++;
        CompletableFuture.supplyAsync(() -> {
            File file = new File(path);
            final long fileLength = file.length();
            Preconditions.assertTrue(fileLength == getFileSizeInBytes(), "Unexpected file size: expected size is " + getFileSizeInBytes() + " but actual size is " + fileLength);
            final Type type = Optional.ofNullable(Type.valueOfIgnoreCase(dataStreamType)).orElseThrow(IllegalStateException::new);
            final TransferType writer = type.getConstructor().apply(path, this);
            try {
                future.complete(writer.transfer(client, routingTable));
            } catch (IOException e) {
                future.completeExceptionally(e);
            }
            return future;
        }, executor);
        fileMap.put(path, future);
    }
    return fileMap;
}

Example 7 with DataStreamReply

use of org.apache.ratis.protocol.DataStreamReply in project incubator-ratis by apache.

the class NettyClientStreamRpc method getClientHandler.

private ChannelInboundHandler getClientHandler() {
    return new ChannelInboundHandlerAdapter() {

        private ClientInvocationId clientInvocationId;

        @Override
        public void channelRead(ChannelHandlerContext ctx, Object msg) {
            if (!(msg instanceof DataStreamReply)) {
                LOG.error("{}: unexpected message {}", this, msg.getClass());
                return;
            }
            final DataStreamReply reply = (DataStreamReply) msg;
            LOG.debug("{}: read {}", this, reply);
            clientInvocationId = ClientInvocationId.valueOf(reply.getClientId(), reply.getStreamId());
            final ReplyQueue queue = reply.isSuccess() ? replies.get(clientInvocationId) : replies.remove(clientInvocationId);
            if (queue != null) {
                final CompletableFuture<DataStreamReply> f = queue.poll();
                if (f != null) {
                    f.complete(reply);
                    if (!reply.isSuccess() && queue.size() > 0) {
                        final IllegalStateException e = new IllegalStateException(this + ": an earlier request failed with " + reply);
                        queue.forEach(future -> future.completeExceptionally(e));
                    }
                    final Integer emptyId = queue.getEmptyId();
                    if (emptyId != null) {
                        timeoutScheduler.onTimeout(replyQueueGracePeriod, // remove the queue if the same queue has been empty for the entire grace period.
                        () -> replies.computeIfPresent(clientInvocationId, (key, q) -> q == queue && emptyId.equals(q.getEmptyId()) ? null : q), LOG, () -> "Timeout check failed, clientInvocationId=" + clientInvocationId);
                    }
                }
            }
        }

        @Override
        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
            LOG.warn(name + ": exceptionCaught", cause);
            Optional.ofNullable(clientInvocationId).map(replies::remove).orElse(ReplyQueue.EMPTY).forEach(f -> f.completeExceptionally(cause));
            LOG.warn(name + ": exceptionCaught", cause);
            ctx.close();
        }
    };
}

Example 8 with DataStreamReply

use of org.apache.ratis.protocol.DataStreamReply in project incubator-ratis by apache.

the class DataStreamTestUtils method writeAndAssertReplies.

static int writeAndAssertReplies(DataStreamOutputImpl out, int bufferSize, int bufferNum) {
    final List<CompletableFuture<DataStreamReply>> futures = new ArrayList<>();
    final List<Integer> sizes = new ArrayList<>();
    // send data
    final int halfBufferSize = bufferSize / 2;
    int dataSize = 0;
    for (int i = 0; i < bufferNum; i++) {
        final int size = halfBufferSize + ThreadLocalRandom.current().nextInt(halfBufferSize);
        sizes.add(size);
        final ByteBuffer bf = initBuffer(dataSize, size);
        futures.add(i == bufferNum - 1 ? out.writeAsync(bf, StandardWriteOption.SYNC) : out.writeAsync(bf));
        dataSize += size;
    }
    {
        // check header
        final DataStreamReply reply = out.getHeaderFuture().join();
        assertSuccessReply(Type.STREAM_HEADER, 0, reply);
    }
    // check writeAsync requests
    for (int i = 0; i < futures.size(); i++) {
        final DataStreamReply reply = futures.get(i).join();
        final Type expectedType = Type.STREAM_DATA;
        assertSuccessReply(expectedType, sizes.get(i).longValue(), reply);
    }
    return dataSize;
}

Example 9 with DataStreamReply

use of org.apache.ratis.protocol.DataStreamReply in project incubator-ratis by apache.

the class DataStreamBaseTest method runTestMockCluster.

void runTestMockCluster(ClientId clientId, int bufferSize, int bufferNum, Exception expectedException, Exception headerException) throws IOException {
    try (final RaftClient client = newRaftClientForDataStream(clientId)) {
        final DataStreamOutputImpl out = (DataStreamOutputImpl) client.getDataStreamApi().stream(null, DataStreamTestUtils.getRoutingTableChainTopology(peers, getPrimaryServer().getPeer()));
        if (headerException != null) {
            final DataStreamReply headerReply = out.getHeaderFuture().join();
            Assert.assertFalse(headerReply.isSuccess());
            final RaftClientReply clientReply = ClientProtoUtils.toRaftClientReply(((DataStreamReplyByteBuffer) headerReply).slice());
            Assert.assertTrue(clientReply.getException().getMessage().contains(headerException.getMessage()));
            return;
        }
        final RaftClientReply clientReply = DataStreamTestUtils.writeAndCloseAndAssertReplies(CollectionUtils.as(servers, Server::getRaftServer), null, out, bufferSize, bufferNum, getPrimaryClientId(), client.getId(), false).join();
        if (expectedException != null) {
            Assert.assertFalse(clientReply.isSuccess());
            Assert.assertTrue(clientReply.getException().getMessage().contains(expectedException.getMessage()));
        } else {
            Assert.assertTrue(clientReply.isSuccess());
        }
    }
}