Examples with XceiverClientReply org.apache.hadoop.hdds.scm.XceiverClientReply used on opensource projects

Example 1 with XceiverClientReply

use of org.apache.hadoop.hdds.scm.XceiverClientReply in project ozone by apache.

the class TestReplicatedFileChecksumHelper method buildValidResponse.

private XceiverClientReply buildValidResponse() {
    // return a GetBlockResponse message of a block and its chunk checksums.
    ContainerProtos.DatanodeBlockID blockID = ContainerProtos.DatanodeBlockID.newBuilder().setContainerID(1).setLocalID(1).setBlockCommitSequenceId(1).build();
    byte[] byteArray = new byte[10];
    ByteString byteString = ByteString.copyFrom(byteArray);
    ContainerProtos.ChecksumData checksumData = ContainerProtos.ChecksumData.newBuilder().setType(CRC32).setBytesPerChecksum(1024).addChecksums(byteString).build();
    ContainerProtos.ChunkInfo chunkInfo = ContainerProtos.ChunkInfo.newBuilder().setChunkName("dummy_chunk").setOffset(1).setLen(10).setChecksumData(checksumData).build();
    ContainerProtos.BlockData blockData = ContainerProtos.BlockData.newBuilder().setBlockID(blockID).addChunks(chunkInfo).build();
    ContainerProtos.GetBlockResponseProto getBlockResponseProto = ContainerProtos.GetBlockResponseProto.newBuilder().setBlockData(blockData).build();
    ContainerProtos.ContainerCommandResponseProto resp = ContainerProtos.ContainerCommandResponseProto.newBuilder().setCmdType(ContainerProtos.Type.GetBlock).setResult(ContainerProtos.Result.SUCCESS).setGetBlock(getBlockResponseProto).build();
    final CompletableFuture<ContainerProtos.ContainerCommandResponseProto> replyFuture = new CompletableFuture<>();
    replyFuture.complete(resp);
    return new XceiverClientReply(replyFuture);
}

Example 2 with XceiverClientReply

use of org.apache.hadoop.hdds.scm.XceiverClientReply in project ozone by apache.

the class LeaderAppendLogEntryGenerator method call.

@Override
public Void call() throws Exception {
    inFlightMessages = new LinkedBlockingQueue<>(inflightLimit);
    OzoneConfiguration conf = createOzoneConfiguration();
    byte[] data = RandomStringUtils.randomAscii(chunkSize).getBytes(StandardCharsets.UTF_8);
    dataToWrite = ByteString.copyFrom(data);
    setServerIdFromFile(conf);
    requestor = RaftPeerProto.newBuilder().setId(RaftPeerId.valueOf(FAKE_FOLLOWER_ID1).toByteString()).setAddress(FAKE_LEADER_ADDDRESS1).build();
    NettyChannelBuilder channelBuilder = NettyChannelBuilder.forTarget(serverAddress);
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
    ManagedChannel build = channelBuilder.build();
    stub = RaftServerProtocolServiceGrpc.newStub(build);
    init();
    if (nextIndex == 0) {
        configureGroup();
    }
    Thread.sleep(3000L);
    XceiverClientRatis client = createXceiverClient(conf);
    client.connect();
    long containerId = 1L;
    System.out.println(client.sendCommand(createContainerRequest(containerId)));
    timer = getMetrics().timer("append-entry");
    runTests(step -> timer.time(() -> {
        inFlightMessages.put(step);
        XceiverClientReply xceiverClientReply = client.sendCommandAsync(createChunkWriteRequest(containerId, step));
        xceiverClientReply.getResponse().thenApply(response -> inFlightMessages.remove(step));
        return null;
    }));
    return null;
}

Example 3 with XceiverClientReply

use of org.apache.hadoop.hdds.scm.XceiverClientReply in project ozone by apache.

the class Test2WayCommitInRatis method test2WayCommitForRetryfailure.

@Test
public void test2WayCommitForRetryfailure() throws Exception {
    OzoneConfiguration conf = new OzoneConfiguration();
    startCluster(conf);
    GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs(XceiverClientRatis.LOG);
    XceiverClientManager clientManager = new XceiverClientManager(conf);
    ContainerWithPipeline container1 = storageContainerLocationClient.allocateContainer(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
    XceiverClientSpi xceiverClient = clientManager.acquireClient(container1.getPipeline());
    Assert.assertEquals(1, xceiverClient.getRefcount());
    Assert.assertEquals(container1.getPipeline(), xceiverClient.getPipeline());
    Pipeline pipeline = xceiverClient.getPipeline();
    XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
    XceiverClientReply reply = xceiverClient.sendCommandAsync(ContainerTestHelper.getCreateContainerRequest(container1.getContainerInfo().getContainerID(), xceiverClient.getPipeline()));
    reply.getResponse().get();
    Assert.assertEquals(3, ratisClient.getCommitInfoMap().size());
    // wait for the container to be created on all the nodes
    xceiverClient.watchForCommit(reply.getLogIndex());
    for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
        // shutdown the ratis follower
        if (RatisTestHelper.isRatisFollower(dn, pipeline)) {
            cluster.shutdownHddsDatanode(dn.getDatanodeDetails());
            break;
        }
    }
    reply = xceiverClient.sendCommandAsync(ContainerTestHelper.getCloseContainer(pipeline, container1.getContainerInfo().getContainerID()));
    reply.getResponse().get();
    xceiverClient.watchForCommit(reply.getLogIndex());
    // commitInfo Map will be reduced to 2 here
    Assert.assertEquals(2, ratisClient.getCommitInfoMap().size());
    clientManager.releaseClient(xceiverClient, false);
    Assert.assertTrue(logCapturer.getOutput().contains("3 way commit failed"));
    Assert.assertTrue(logCapturer.getOutput().contains("Committed by majority"));
    logCapturer.stopCapturing();
    shutdown();
}

Example 4 with XceiverClientReply

use of org.apache.hadoop.hdds.scm.XceiverClientReply in project ozone by apache.

the class TestCommitWatcher method testReleaseBuffers.

@Test
public void testReleaseBuffers() throws Exception {
    int capacity = 2;
    BufferPool bufferPool = new BufferPool(chunkSize, capacity);
    XceiverClientManager clientManager = new XceiverClientManager(conf);
    ContainerWithPipeline container = storageContainerLocationClient.allocateContainer(HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
    Pipeline pipeline = container.getPipeline();
    long containerId = container.getContainerInfo().getContainerID();
    XceiverClientSpi xceiverClient = clientManager.acquireClient(pipeline);
    Assert.assertEquals(1, xceiverClient.getRefcount());
    Assert.assertTrue(xceiverClient instanceof XceiverClientRatis);
    XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
    CommitWatcher watcher = new CommitWatcher(bufferPool, ratisClient);
    BlockID blockID = ContainerTestHelper.getTestBlockID(containerId);
    List<XceiverClientReply> replies = new ArrayList<>();
    long length = 0;
    List<CompletableFuture<ContainerProtos.ContainerCommandResponseProto>> futures = new ArrayList<>();
    for (int i = 0; i < capacity; i++) {
        ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest(pipeline, blockID, chunkSize, null);
        // add the data to the buffer pool
        final ChunkBuffer byteBuffer = bufferPool.allocateBuffer(0);
        byteBuffer.put(writeChunkRequest.getWriteChunk().getData());
        ratisClient.sendCommandAsync(writeChunkRequest);
        ContainerProtos.ContainerCommandRequestProto putBlockRequest = ContainerTestHelper.getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
        XceiverClientReply reply = ratisClient.sendCommandAsync(putBlockRequest);
        final List<ChunkBuffer> bufferList = singletonList(byteBuffer);
        length += byteBuffer.position();
        CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future = reply.getResponse().thenApply(v -> {
            watcher.updateCommitInfoMap(reply.getLogIndex(), bufferList);
            return v;
        });
        futures.add(future);
        watcher.getFutureMap().put(length, future);
        replies.add(reply);
    }
    Assert.assertTrue(replies.size() == 2);
    // wait on the 1st putBlock to complete
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future1 = futures.get(0);
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future2 = futures.get(1);
    future1.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) chunkSize).equals(future1));
    // wait on 2nd putBlock to complete
    future2.get();
    Assert.assertNotNull(watcher.getFutureMap().get((long) 2 * chunkSize));
    Assert.assertTrue(watcher.getFutureMap().get((long) 2 * chunkSize).equals(future2));
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 2);
    watcher.watchOnFirstIndex();
    Assert.assertFalse(watcher.getCommitIndex2flushedDataMap().containsKey(replies.get(0).getLogIndex()));
    Assert.assertFalse(watcher.getFutureMap().containsKey(chunkSize));
    Assert.assertTrue(watcher.getTotalAckDataLength() >= chunkSize);
    watcher.watchOnLastIndex();
    Assert.assertFalse(watcher.getCommitIndex2flushedDataMap().containsKey(replies.get(1).getLogIndex()));
    Assert.assertFalse(watcher.getFutureMap().containsKey(2 * chunkSize));
    Assert.assertTrue(watcher.getTotalAckDataLength() == 2 * chunkSize);
    Assert.assertTrue(watcher.getFutureMap().isEmpty());
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().isEmpty());
}

Example 5 with XceiverClientReply

use of org.apache.hadoop.hdds.scm.XceiverClientReply in project ozone by apache.

the class CommitWatcher method watchForCommit.

/**
 * calls watchForCommit API of the Ratis Client. For Standalone client,
 * it is a no op.
 * @param commitIndex log index to watch for
 * @return minimum commit index replicated to all nodes
 * @throws IOException IOException in case watch gets timed out
 */
public XceiverClientReply watchForCommit(long commitIndex) throws IOException {
    long index;
    try {
        XceiverClientReply reply = xceiverClient.watchForCommit(commitIndex);
        if (reply == null) {
            index = 0;
        } else {
            index = reply.getLogIndex();
        }
        adjustBuffers(index);
        return reply;
    } catch (InterruptedException e) {
        // Re-interrupt the thread while catching InterruptedException
        Thread.currentThread().interrupt();
        throw getIOExceptionForWatchForCommit(commitIndex, e);
    } catch (TimeoutException | ExecutionException e) {
        throw getIOExceptionForWatchForCommit(commitIndex, e);
    }
}