Examples with RecordMetadata org.apache.kafka.clients.producer.RecordMetadata used on opensource projects

Example 61 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testIdempotenceWithMultipleInflights.

@Test
public void testIdempotenceWithMultipleInflights() throws Exception {
    final long producerId = 343434L;
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
    // Send first ProduceRequest
    Future<RecordMetadata> request1 = appendToAccumulator(tp0);
    sender.runOnce();
    String nodeId = client.requests().peek().destination();
    Node node = new Node(Integer.valueOf(nodeId), "localhost", 0);
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(1, transactionManager.sequenceNumber(tp0).longValue());
    assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
    // Send second ProduceRequest
    Future<RecordMetadata> request2 = appendToAccumulator(tp0);
    sender.runOnce();
    assertEquals(2, client.inFlightRequestCount());
    assertEquals(2, transactionManager.sequenceNumber(tp0).longValue());
    assertEquals(OptionalInt.empty(), transactionManager.lastAckedSequence(tp0));
    assertFalse(request1.isDone());
    assertFalse(request2.isDone());
    assertTrue(client.isReady(node, time.milliseconds()));
    sendIdempotentProducerResponse(0, tp0, Errors.NONE, 0L);
    // receive response 0
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
    assertTrue(request1.isDone());
    assertEquals(0, request1.get().offset());
    assertFalse(request2.isDone());
    sendIdempotentProducerResponse(1, tp0, Errors.NONE, 1L);
    // receive response 1
    sender.runOnce();
    assertEquals(OptionalInt.of(1), transactionManager.lastAckedSequence(tp0));
    assertFalse(client.hasInFlightRequests());
    assertEquals(0, sender.inFlightBatches(tp0).size());
    assertTrue(request2.isDone());
    assertEquals(1, request2.get().offset());
}

Example 62 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testExpiryOfFirstBatchShouldCauseEpochBumpIfFutureBatchesFail.

@Test
public void testExpiryOfFirstBatchShouldCauseEpochBumpIfFutureBatchesFail() throws Exception {
    final long producerId = 343434L;
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
    // Send first ProduceRequest
    Future<RecordMetadata> request1 = appendToAccumulator(tp0);
    // send request
    sender.runOnce();
    time.sleep(1000L);
    Future<RecordMetadata> request2 = appendToAccumulator(tp0);
    // send request
    sender.runOnce();
    assertEquals(2, client.inFlightRequestCount());
    sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_OR_FOLLOWER, -1);
    // receive first response
    sender.runOnce();
    Node node = metadata.fetch().nodes().get(0);
    time.sleep(1000L);
    client.disconnect(node.idString());
    client.backoff(node, 10);
    // now expire the first batch.
    sender.runOnce();
    assertFutureFailure(request1, TimeoutException.class);
    assertTrue(transactionManager.hasUnresolvedSequence(tp0));
    // let's enqueue another batch, which should not be dequeued until the unresolved state is clear.
    appendToAccumulator(tp0);
    time.sleep(20);
    assertFalse(request2.isDone());
    // send second request
    sender.runOnce();
    sendIdempotentProducerResponse(1, tp0, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER, 1);
    // receive second response, the third request shouldn't be sent since we are in an unresolved state.
    sender.runOnce();
    Deque<ProducerBatch> batches = accumulator.batches().get(tp0);
    // The epoch should be bumped and the second request should be requeued
    assertEquals(2, batches.size());
    sender.runOnce();
    assertEquals((short) 1, transactionManager.producerIdAndEpoch().epoch);
    assertEquals(1, transactionManager.sequenceNumber(tp0).longValue());
    assertFalse(transactionManager.hasUnresolvedSequence(tp0));
}

Example 63 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testSplitBatchAndSend.

@SuppressWarnings("deprecation")
private void testSplitBatchAndSend(TransactionManager txnManager, ProducerIdAndEpoch producerIdAndEpoch, TopicPartition tp) throws Exception {
    int maxRetries = 1;
    String topic = tp.topic();
    int deliveryTimeoutMs = 3000;
    long totalSize = 1024 * 1024;
    String metricGrpName = "producer-metrics";
    // Set a good compression ratio.
    CompressionRatioEstimator.setEstimation(topic, CompressionType.GZIP, 0.2f);
    try (Metrics m = new Metrics()) {
        accumulator = new RecordAccumulator(logContext, batchSize, CompressionType.GZIP, 0, 0L, deliveryTimeoutMs, m, metricGrpName, time, new ApiVersions(), txnManager, new BufferPool(totalSize, batchSize, metrics, time, "producer-internal-metrics"));
        SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
        Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 1000L, txnManager, new ApiVersions());
        // Create a two broker cluster, with partition 0 on broker 0 and partition 1 on broker 1
        MetadataResponse metadataUpdate1 = RequestTestUtils.metadataUpdateWith(2, Collections.singletonMap(topic, 2));
        client.prepareMetadataUpdate(metadataUpdate1);
        // Send the first message.
        long nowMs = time.milliseconds();
        Future<RecordMetadata> f1 = accumulator.append(tp, 0L, "key1".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false, nowMs).future;
        Future<RecordMetadata> f2 = accumulator.append(tp, 0L, "key2".getBytes(), new byte[batchSize / 2], null, null, MAX_BLOCK_TIMEOUT, false, nowMs).future;
        // connect
        sender.runOnce();
        // send produce request
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence should be 2");
        String id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        Node node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        Map<TopicPartition, ProduceResponse.PartitionResponse> responseMap = new HashMap<>();
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.MESSAGE_TOO_LARGE));
        client.respond(new ProduceResponse(responseMap));
        // split and reenqueue
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence should be 2");
        // The compression ratio should have been improved once.
        assertEquals(CompressionType.GZIP.rate - CompressionRatioEstimator.COMPRESSION_RATIO_IMPROVING_STEP, CompressionRatioEstimator.estimation(topic, CompressionType.GZIP), 0.01);
        // send the first produce request
        sender.runOnce();
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should be 2");
        assertFalse(f1.isDone(), "The future shouldn't have been done.");
        assertFalse(f2.isDone(), "The future shouldn't have been done.");
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 0L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 0, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.runOnce();
        assertTrue(f1.isDone(), "The future should have been done.");
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should still be 2");
        assertEquals(OptionalInt.of(0), txnManager.lastAckedSequence(tp), "The last ack'd sequence number should be 0");
        assertFalse(f2.isDone(), "The future shouldn't have been done.");
        assertEquals(0L, f1.get().offset(), "Offset of the first message should be 0");
        // send the seconcd produce request
        sender.runOnce();
        id = client.requests().peek().destination();
        assertEquals(ApiKeys.PRODUCE, client.requests().peek().requestBuilder().apiKey());
        node = new Node(Integer.valueOf(id), "localhost", 0);
        assertEquals(1, client.inFlightRequestCount());
        assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
        responseMap.put(tp, new ProduceResponse.PartitionResponse(Errors.NONE, 1L, 0L, 0L));
        client.respond(produceRequestMatcher(tp, producerIdAndEpoch, 1, txnManager.isTransactional()), new ProduceResponse(responseMap));
        // receive
        sender.runOnce();
        assertTrue(f2.isDone(), "The future should have been done.");
        assertEquals(2, txnManager.sequenceNumber(tp).longValue(), "The next sequence number should be 2");
        assertEquals(OptionalInt.of(1), txnManager.lastAckedSequence(tp), "The last ack'd sequence number should be 1");
        assertEquals(1L, f2.get().offset(), "Offset of the first message should be 1");
        assertTrue(accumulator.batches().get(tp).isEmpty(), "There should be no batch in the accumulator");
        assertTrue((Double) (m.metrics().get(senderMetrics.batchSplitRate).metricValue()) > 0, "There should be a split");
    }
}

Example 64 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testMessageFormatDownConversion.

@Test
public void testMessageFormatDownConversion() throws Exception {
    // this test case verifies the behavior when the version of the produce request supported by the
    // broker changes after the record set is created
    long offset = 0;
    // start off support produce request v3
    apiVersions.update("0", NodeApiVersions.create());
    Future<RecordMetadata> future = appendToAccumulator(tp0, 0L, "key", "value");
    // now the partition leader supports only v2
    apiVersions.update("0", NodeApiVersions.create(ApiKeys.PRODUCE.id, (short) 0, (short) 2));
    client.prepareResponse(body -> {
        ProduceRequest request = (ProduceRequest) body;
        if (request.version() != 2)
            return false;
        MemoryRecords records = partitionRecords(request).get(tp0);
        return records != null && records.sizeInBytes() > 0 && records.hasMatchingMagic(RecordBatch.MAGIC_VALUE_V1);
    }, produceResponse(tp0, offset, Errors.NONE, 0));
    // connect
    sender.runOnce();
    // send produce request
    sender.runOnce();
    assertTrue(future.isDone(), "Request should be completed");
    assertEquals(offset, future.get().offset());
}

Example 65 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testRetryWhenProducerIdChanges.

@Test
public void testRetryWhenProducerIdChanges() throws InterruptedException {
    final long producerId = 343434L;
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Short.MAX_VALUE, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    int maxRetries = 10;
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
    Future<RecordMetadata> responseFuture = appendToAccumulator(tp0);
    // connect.
    sender.runOnce();
    // send.
    sender.runOnce();
    String id = client.requests().peek().destination();
    Node node = new Node(Integer.valueOf(id), "localhost", 0);
    assertEquals(1, client.inFlightRequestCount());
    assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
    client.disconnect(id);
    assertEquals(0, client.inFlightRequestCount());
    assertFalse(client.isReady(node, time.milliseconds()), "Client ready status should be false");
    // receive error
    sender.runOnce();
    // reset producer ID because epoch is maxed out
    sender.runOnce();
    prepareAndReceiveInitProducerId(producerId + 1, Errors.NONE);
    // nothing to do, since the pid has changed. We should check the metrics for errors.
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount(), "Expected requests to be retried after pid change");
    assertFalse(responseFuture.isDone());
    assertEquals(1, (long) transactionManager.sequenceNumber(tp0));
}