Examples with RecordMetadata org.apache.kafka.clients.producer.RecordMetadata used on opensource projects

Example 51 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testSequenceNumberIncrement.

@Test
public void testSequenceNumberIncrement() throws InterruptedException {
    final long producerId = 343434L;
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    int maxRetries = 10;
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
    Future<RecordMetadata> responseFuture = appendToAccumulator(tp0);
    client.prepareResponse(body -> {
        if (body instanceof ProduceRequest) {
            ProduceRequest request = (ProduceRequest) body;
            MemoryRecords records = partitionRecords(request).get(tp0);
            Iterator<MutableRecordBatch> batchIterator = records.batches().iterator();
            assertTrue(batchIterator.hasNext());
            RecordBatch batch = batchIterator.next();
            assertFalse(batchIterator.hasNext());
            assertEquals(0, batch.baseSequence());
            assertEquals(producerId, batch.producerId());
            assertEquals(0, batch.producerEpoch());
            return true;
        }
        return false;
    }, produceResponse(tp0, 0, Errors.NONE, 0));
    // connect.
    sender.runOnce();
    // send.
    sender.runOnce();
    // receive response
    sender.runOnce();
    assertTrue(responseFuture.isDone());
    assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
    assertEquals(1L, (long) transactionManager.sequenceNumber(tp0));
}

Example 52 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testUnresolvedSequencesAreNotFatal.

@Test
public void testUnresolvedSequencesAreNotFatal() throws Exception {
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(123456L, (short) 0);
    apiVersions.update("0", NodeApiVersions.create(ApiKeys.INIT_PRODUCER_ID.id, (short) 0, (short) 3));
    TransactionManager txnManager = new TransactionManager(logContext, "testUnresolvedSeq", 60000, 100, apiVersions);
    setupWithTransactionState(txnManager);
    doInitTransactions(txnManager, producerIdAndEpoch);
    txnManager.beginTransaction();
    txnManager.maybeAddPartition(tp0);
    client.prepareResponse(new AddPartitionsToTxnResponse(0, Collections.singletonMap(tp0, Errors.NONE)));
    sender.runOnce();
    // Send first ProduceRequest
    Future<RecordMetadata> request1 = appendToAccumulator(tp0);
    // send request
    sender.runOnce();
    time.sleep(1000L);
    appendToAccumulator(tp0);
    // send request
    sender.runOnce();
    assertEquals(2, client.inFlightRequestCount());
    sendIdempotentProducerResponse(0, tp0, Errors.NOT_LEADER_OR_FOLLOWER, -1);
    // receive first response
    sender.runOnce();
    Node node = metadata.fetch().nodes().get(0);
    time.sleep(1000L);
    client.disconnect(node.idString());
    client.backoff(node, 10);
    // now expire the first batch.
    sender.runOnce();
    assertFutureFailure(request1, TimeoutException.class);
    assertTrue(txnManager.hasUnresolvedSequence(tp0));
    // Loop once and confirm that the transaction manager does not enter a fatal error state
    sender.runOnce();
    assertTrue(txnManager.hasAbortableError());
}

Example 53 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testCanRetryWithoutIdempotence.

@Test
public void testCanRetryWithoutIdempotence() throws Exception {
    // do a successful retry
    Future<RecordMetadata> future = appendToAccumulator(tp0, 0L, "key", "value");
    // connect
    sender.runOnce();
    // send produce request
    sender.runOnce();
    String id = client.requests().peek().destination();
    Node node = new Node(Integer.parseInt(id), "localhost", 0);
    assertEquals(1, client.inFlightRequestCount());
    assertTrue(client.hasInFlightRequests());
    assertEquals(1, sender.inFlightBatches(tp0).size());
    assertTrue(client.isReady(node, time.milliseconds()), "Client ready status should be true");
    assertFalse(future.isDone());
    client.respond(body -> {
        ProduceRequest request = (ProduceRequest) body;
        assertFalse(RequestTestUtils.hasIdempotentRecords(request));
        return true;
    }, produceResponse(tp0, -1L, Errors.TOPIC_AUTHORIZATION_FAILED, 0));
    sender.runOnce();
    assertTrue(future.isDone());
    try {
        future.get();
    } catch (Exception e) {
        assertTrue(e.getCause() instanceof TopicAuthorizationException);
    }
}

Example 54 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testExpiryOfUnsentBatchesShouldNotCauseUnresolvedSequences.

@Test
public void testExpiryOfUnsentBatchesShouldNotCauseUnresolvedSequences() throws Exception {
    final long producerId = 343434L;
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(producerId, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    assertEquals(0, transactionManager.sequenceNumber(tp0).longValue());
    // Send first ProduceRequest
    Future<RecordMetadata> request1 = appendToAccumulator(tp0, 0L, "key", "value");
    Node node = metadata.fetch().nodes().get(0);
    time.sleep(10000L);
    client.disconnect(node.idString());
    client.backoff(node, 10);
    sender.runOnce();
    assertFutureFailure(request1, TimeoutException.class);
    assertFalse(transactionManager.hasUnresolvedSequence(tp0));
}

Example 55 with RecordMetadata

use of org.apache.kafka.clients.producer.RecordMetadata in project kafka by apache.

the class SenderTest method testForceCloseWithProducerIdReset.

@Test
public void testForceCloseWithProducerIdReset() throws Exception {
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager);
    prepareAndReceiveInitProducerId(1L, Short.MAX_VALUE, Errors.NONE);
    assertTrue(transactionManager.hasProducerId());
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, 10, senderMetrics, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, transactionManager, apiVersions);
    Future<RecordMetadata> failedResponse = appendToAccumulator(tp0);
    Future<RecordMetadata> successfulResponse = appendToAccumulator(tp1);
    // connect and send.
    sender.runOnce();
    assertEquals(1, client.inFlightRequestCount());
    Map<TopicPartition, OffsetAndError> responses = new LinkedHashMap<>();
    responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_OR_FOLLOWER));
    responses.put(tp0, new OffsetAndError(-1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER));
    client.respond(produceResponse(responses));
    // out of order sequence error triggers producer ID reset because epoch is maxed out
    sender.runOnce();
    // initiate force close
    sender.forceClose();
    // this should not block
    sender.runOnce();
    // run main loop to test forceClose flag
    sender.run();
    assertFalse(accumulator.hasUndrained(), "Pending batches are not aborted.");
    assertTrue(successfulResponse.isDone());
}