Examples with ProducerIdAndEpoch org.apache.kafka.common.utils.ProducerIdAndEpoch used on opensource projects

Example 16 with ProducerIdAndEpoch

use of org.apache.kafka.common.utils.ProducerIdAndEpoch in project kafka by apache.

the class SenderTest method testIdempotentInitProducerIdWithMaxInFlightOne.

/**
 * Verifies that InitProducerId of idempotent producer succeeds even if metadata requests
 * are pending with only one bootstrap node available and maxInFlight=1, where multiple
 * polls are necessary to send requests.
 */
@Test
public void testIdempotentInitProducerIdWithMaxInFlightOne() throws Exception {
    final long producerId = 123456L;
    createMockClientWithMaxFlightOneMetadataPending();
    // Initialize transaction manager. InitProducerId will be queued up until metadata response
    // is processed.
    TransactionManager transactionManager = createTransactionManager();
    setupWithTransactionState(transactionManager, false, null, false);
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, (short) 0);
    // Process metadata and InitProducerId responses.
    // Verify producerId after the sender is run to process responses.
    MetadataResponse metadataUpdate = RequestTestUtils.metadataUpdateWith(1, Collections.emptyMap());
    client.respond(metadataUpdate);
    sender.runOnce();
    sender.runOnce();
    client.respond(initProducerIdResponse(producerIdAndEpoch.producerId, producerIdAndEpoch.epoch, Errors.NONE));
    waitForProducerId(transactionManager, producerIdAndEpoch);
}

Example 17 with ProducerIdAndEpoch

use of org.apache.kafka.common.utils.ProducerIdAndEpoch in project kafka by apache.

the class TransactionManagerTest method testEpochBumpAfterLastInflightBatchFails.

@Test
public void testEpochBumpAfterLastInflightBatchFails() {
    initializeTransactionManager(Optional.empty());
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
    initializeIdempotentProducerId(producerId, epoch);
    TopicPartition tp0 = new TopicPartition("foo", 0);
    ProducerBatch b1 = writeIdempotentBatchWithValue(transactionManager, tp0, "1");
    ProducerBatch b2 = writeIdempotentBatchWithValue(transactionManager, tp0, "2");
    ProducerBatch b3 = writeIdempotentBatchWithValue(transactionManager, tp0, "3");
    assertEquals(Integer.valueOf(3), transactionManager.sequenceNumber(tp0));
    // The first batch fails with a timeout
    transactionManager.markSequenceUnresolved(b1);
    transactionManager.handleFailedBatch(b1, new TimeoutException(), false);
    assertTrue(transactionManager.hasUnresolvedSequences());
    // The second batch succeeds, but sequence numbers are still not resolved
    transactionManager.handleCompletedBatch(b2, new ProduceResponse.PartitionResponse(Errors.NONE, 500L, time.milliseconds(), 0L));
    transactionManager.bumpIdempotentEpochAndResetIdIfNeeded();
    assertEquals(producerIdAndEpoch, transactionManager.producerIdAndEpoch());
    assertTrue(transactionManager.hasUnresolvedSequences());
    // When the last inflight batch fails, we have to bump the epoch
    transactionManager.handleFailedBatch(b3, new TimeoutException(), false);
    // Run sender loop to trigger epoch bump
    runUntil(() -> transactionManager.producerIdAndEpoch().epoch == 2);
    assertFalse(transactionManager.hasUnresolvedSequences());
    assertEquals(0, transactionManager.sequenceNumber(tp0).intValue());
}

Example 18 with ProducerIdAndEpoch

use of org.apache.kafka.common.utils.ProducerIdAndEpoch in project kafka by apache.

the class RecordAccumulatorTest method testRecordsDrainedWhenTransactionCompleting.

@Test
public void testRecordsDrainedWhenTransactionCompleting() throws Exception {
    int batchSize = 1025;
    int deliveryTimeoutMs = 3200;
    int lingerMs = 10;
    long totalSize = 10 * batchSize;
    TransactionManager transactionManager = Mockito.mock(TransactionManager.class);
    RecordAccumulator accumulator = createTestRecordAccumulator(transactionManager, deliveryTimeoutMs, batchSize, totalSize, CompressionType.NONE, lingerMs);
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(12345L, (short) 5);
    Mockito.when(transactionManager.producerIdAndEpoch()).thenReturn(producerIdAndEpoch);
    Mockito.when(transactionManager.isSendToPartitionAllowed(tp1)).thenReturn(true);
    Mockito.when(transactionManager.isPartitionAdded(tp1)).thenReturn(true);
    Mockito.when(transactionManager.firstInFlightSequence(tp1)).thenReturn(0);
    // Initially, the transaction is still in progress, so we should respect the linger.
    Mockito.when(transactionManager.isCompleting()).thenReturn(false);
    accumulator.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false, time.milliseconds());
    accumulator.append(tp1, 0L, key, value, Record.EMPTY_HEADERS, null, maxBlockTimeMs, false, time.milliseconds());
    assertTrue(accumulator.hasUndrained());
    RecordAccumulator.ReadyCheckResult firstResult = accumulator.ready(cluster, time.milliseconds());
    assertEquals(0, firstResult.readyNodes.size());
    Map<Integer, List<ProducerBatch>> firstDrained = accumulator.drain(cluster, firstResult.readyNodes, Integer.MAX_VALUE, time.milliseconds());
    assertEquals(0, firstDrained.size());
    // Once the transaction begins completion, then the batch should be drained immediately.
    Mockito.when(transactionManager.isCompleting()).thenReturn(true);
    RecordAccumulator.ReadyCheckResult secondResult = accumulator.ready(cluster, time.milliseconds());
    assertEquals(1, secondResult.readyNodes.size());
    Node readyNode = secondResult.readyNodes.iterator().next();
    Map<Integer, List<ProducerBatch>> secondDrained = accumulator.drain(cluster, secondResult.readyNodes, Integer.MAX_VALUE, time.milliseconds());
    assertEquals(Collections.singleton(readyNode.id()), secondDrained.keySet());
    List<ProducerBatch> batches = secondDrained.get(readyNode.id());
    assertEquals(1, batches.size());
}

Example 19 with ProducerIdAndEpoch

use of org.apache.kafka.common.utils.ProducerIdAndEpoch in project kafka by apache.

the class TransactionManagerTest method testBumpEpochAfterTimeoutWithoutPendingInflightRequests.

@Test
public void testBumpEpochAfterTimeoutWithoutPendingInflightRequests() {
    initializeTransactionManager(Optional.empty());
    long producerId = 15L;
    short epoch = 5;
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
    initializeIdempotentProducerId(producerId, epoch);
    // Nothing to resolve, so no reset is needed
    transactionManager.bumpIdempotentEpochAndResetIdIfNeeded();
    assertEquals(producerIdAndEpoch, transactionManager.producerIdAndEpoch());
    TopicPartition tp0 = new TopicPartition("foo", 0);
    assertEquals(Integer.valueOf(0), transactionManager.sequenceNumber(tp0));
    ProducerBatch b1 = writeIdempotentBatchWithValue(transactionManager, tp0, "1");
    assertEquals(Integer.valueOf(1), transactionManager.sequenceNumber(tp0));
    transactionManager.handleCompletedBatch(b1, new ProduceResponse.PartitionResponse(Errors.NONE, 500L, time.milliseconds(), 0L));
    assertEquals(OptionalInt.of(0), transactionManager.lastAckedSequence(tp0));
    // Marking sequence numbers unresolved without inflight requests is basically a no-op.
    transactionManager.markSequenceUnresolved(b1);
    transactionManager.maybeResolveSequences();
    assertEquals(producerIdAndEpoch, transactionManager.producerIdAndEpoch());
    assertFalse(transactionManager.hasUnresolvedSequences());
    // We have a new batch which fails with a timeout
    ProducerBatch b2 = writeIdempotentBatchWithValue(transactionManager, tp0, "2");
    assertEquals(Integer.valueOf(2), transactionManager.sequenceNumber(tp0));
    transactionManager.markSequenceUnresolved(b2);
    transactionManager.handleFailedBatch(b2, new TimeoutException(), false);
    assertTrue(transactionManager.hasUnresolvedSequences());
    // We only had one inflight batch, so we should be able to clear the unresolved status
    // and bump the epoch
    transactionManager.maybeResolveSequences();
    assertFalse(transactionManager.hasUnresolvedSequences());
    // Run sender loop to trigger epoch bump
    runUntil(() -> transactionManager.producerIdAndEpoch().epoch == 6);
}

Example 20 with ProducerIdAndEpoch

use of org.apache.kafka.common.utils.ProducerIdAndEpoch in project kafka by apache.

the class TransactionManagerTest method testNoProducerIdResetAfterLastInFlightBatchSucceeds.

@Test
public void testNoProducerIdResetAfterLastInFlightBatchSucceeds() {
    initializeTransactionManager(Optional.empty());
    long producerId = 15L;
    short epoch = 5;
    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(producerId, epoch);
    initializeIdempotentProducerId(producerId, epoch);
    TopicPartition tp0 = new TopicPartition("foo", 0);
    ProducerBatch b1 = writeIdempotentBatchWithValue(transactionManager, tp0, "1");
    ProducerBatch b2 = writeIdempotentBatchWithValue(transactionManager, tp0, "2");
    ProducerBatch b3 = writeIdempotentBatchWithValue(transactionManager, tp0, "3");
    assertEquals(3, transactionManager.sequenceNumber(tp0).intValue());
    // The first batch fails with a timeout
    transactionManager.markSequenceUnresolved(b1);
    transactionManager.handleFailedBatch(b1, new TimeoutException(), false);
    assertTrue(transactionManager.hasUnresolvedSequences());
    // The reset should not occur until sequence numbers have been resolved
    transactionManager.bumpIdempotentEpochAndResetIdIfNeeded();
    assertEquals(producerIdAndEpoch, transactionManager.producerIdAndEpoch());
    assertTrue(transactionManager.hasUnresolvedSequences());
    // The second batch fails as well with a timeout
    transactionManager.handleFailedBatch(b2, new TimeoutException(), false);
    transactionManager.bumpIdempotentEpochAndResetIdIfNeeded();
    assertEquals(producerIdAndEpoch, transactionManager.producerIdAndEpoch());
    assertTrue(transactionManager.hasUnresolvedSequences());
    // The third batch succeeds, which should resolve the sequence number without
    // requiring a producerId reset.
    transactionManager.handleCompletedBatch(b3, new ProduceResponse.PartitionResponse(Errors.NONE, 500L, time.milliseconds(), 0L));
    transactionManager.maybeResolveSequences();
    assertEquals(producerIdAndEpoch, transactionManager.producerIdAndEpoch());
    assertFalse(transactionManager.hasUnresolvedSequences());
    assertEquals(3, transactionManager.sequenceNumber(tp0).intValue());
}