Examples with ClientRequest org.apache.kafka.clients.ClientRequest used on opensource projects

Example 36 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project kafka by apache.

the class SenderTest method createMockClientWithMaxFlightOneMetadataPending.

private void createMockClientWithMaxFlightOneMetadataPending() {
    client = new MockClient(time, metadata) {

        volatile boolean canSendMore = true;

        @Override
        public Node leastLoadedNode(long now) {
            for (Node node : metadata.fetch().nodes()) {
                if (isReady(node, now) && canSendMore)
                    return node;
            }
            return null;
        }

        @Override
        public List<ClientResponse> poll(long timeoutMs, long now) {
            canSendMore = inFlightRequestCount() < 1;
            return super.poll(timeoutMs, now);
        }
    };
    // Send metadata request and wait until request is sent. `leastLoadedNode` will be null once
    // request is in progress since no more requests can be sent to the node. Node will be ready
    // on the next poll() after response is processed later on in tests which use this method.
    MetadataRequest.Builder builder = new MetadataRequest.Builder(Collections.emptyList(), false);
    Node node = metadata.fetch().nodes().get(0);
    ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true);
    while (!client.ready(node, time.milliseconds())) client.poll(0, time.milliseconds());
    client.send(request, time.milliseconds());
    while (client.leastLoadedNode(time.milliseconds()) != null) client.poll(0, time.milliseconds());
}

Example 37 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project kafka by apache.

the class SenderTest method testCorrectHandlingOfOutOfOrderResponses.

@Test
public void testCorrectHandlingOfOutOfOrderResponses() throws Exception {
    final long producerId = 343434L;
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
    // Send first ProduceRequest
    Future<RecordMetadata> request1 = appendToAccumulator(tp0);
    sender.runOnce();
    String nodeId = client.requests().peek().destination();
    Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(1, transactionManager.sequenceNumber(tp0).longValue());
    assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
    // Send second ProduceRequest
    Future<RecordMetadata> request2 = appendToAccumulator(tp0);
    sender.runOnce();
    assertEquals(2, client.inFlightRequestCount());
    assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
    assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
    assertFalse(request1.isDone());
    assertFalse(request2.isDone());
    assertTrue(client.isReady(node, time.milliseconds()));
    ClientRequest firstClientRequest = client.requests().peek();
    ClientRequest secondClientRequest = (ClientRequest) client.requests().toArray()[1];
    client.respondToRequest(secondClientRequest, produceResponse(tp0, -1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, -1));
    // receive response 1
    sender.runOnce();
    Deque<ProducerBatch> queuedBatches = accumulator.batches().get(tp0);
    // Make sure that we are queueing the second batch first.
    assertEquals(1, queuedBatches.size());
    assertEquals(1, queuedBatches.peekFirst().baseSequence());
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
    client.respondToRequest(firstClientRequest, produceResponse(tp0, -1, Errors.NOT_LEADER_OR_FOLLOWER, -1));
    // receive response 0
    sender.runOnce();
    // Make sure we requeued both batches in the correct order.
    assertEquals(2, queuedBatches.size());
    assertEquals(0, queuedBatches.peekFirst().baseSequence());
    assertEquals(1, queuedBatches.peekLast().baseSequence());
    assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
    assertEquals(0, client.inFlightRequestCount());
    assertFalse(request1.isDone());
    assertFalse(request2.isDone());
    // send request 0
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    // don't do anything, only one inflight allowed once we are retrying.
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
    // Make sure that the requests are sent in order, even though the previous responses were not in order.
    sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L);
    // receive response 0
    sender.runOnce();
    assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
    assertEquals(0, client.inFlightRequestCount());
    assertTrue(request1.isDone());
    assertEquals(0, request1.get().offset());
    // send request 1
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L);
    // receive response 1
    sender.runOnce();
    assertFalse(client.hasInFlightRequests());
    assertEquals(OptionalInt.of(1), transactionManager.lastAckedSequence(tp0));
    assertTrue(request2.isDone());
    assertEquals(1, request2.get().offset());
}

Example 38 with ClientRequest

use of org.apache.kafka.clients.ClientRequest in project kafka by apache.

the class SenderTest method testCorrectHandlingOfOutOfOrderResponsesWhenSecondSucceeds.

@Test
public void testCorrectHandlingOfOutOfOrderResponsesWhenSecondSucceeds() throws Exception {
    final long producerId = 343434L;
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
    // Send first ProduceRequest
    Future<RecordMetadata> request1 = appendToAccumulator(tp0);
    sender.runOnce();
    String nodeId = client.requests().peek().destination();
    Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
    assertEquals(1, client.inFlightRequestCount());
    // Send second ProduceRequest
    Future<RecordMetadata> request2 = appendToAccumulator(tp0);
    sender.runOnce();
    assertEquals(2, client.inFlightRequestCount());
    assertFalse(request1.isDone());
    assertFalse(request2.isDone());
    assertTrue(client.isReady(node, time.milliseconds()));
    ClientRequest firstClientRequest = client.requests().peek();
    ClientRequest secondClientRequest = (ClientRequest) client.requests().toArray()[1];
    client.respondToRequest(secondClientRequest, produceResponse(tp0, 1, Errors.NONE, 1));
    // receive response 1
    sender.runOnce();
    assertTrue(request2.isDone());
    assertEquals(1, request2.get().offset());
    assertFalse(request1.isDone());
    Deque<ProducerBatch> queuedBatches = accumulator.batches().get(tp0);
    assertEquals(0, queuedBatches.size());
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(OptionalInt.of(1), transactionManager.lastAckedSequence(tp0));
    client.respondToRequest(firstClientRequest, produceResponse(tp0, -1, Errors.REQUEST_TIMED_OUT, -1));
    // receive response 0
    sender.runOnce();
    // Make sure we requeued both batches in the correct order.
    assertEquals(1, queuedBatches.size());
    assertEquals(0, queuedBatches.peekFirst().baseSequence());
    assertEquals(OptionalInt.of(1), transactionManager.lastAckedSequence(tp0));
    assertEquals(0, client.inFlightRequestCount());
    // resend request 0
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(OptionalInt.of(1), transactionManager.lastAckedSequence(tp0));
    // Make sure we handle the out of order successful responses correctly.
    sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L);
    // receive response 0
    sender.runOnce();
    assertEquals(0, queuedBatches.size());
    assertEquals(OptionalInt.of(1), transactionManager.lastAckedSequence(tp0));
    assertEquals(0, client.inFlightRequestCount());
    assertFalse(client.hasInFlightRequests());
    assertTrue(request1.isDone());
    assertEquals(0, request1.get().offset());
}