Examples with ConsumerGroupMetadata org.apache.kafka.clients.consumer.ConsumerGroupMetadata used on opensource projects

Example 36 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class MockProducerTest method shouldResetSentOffsetsFlagOnlyWhenBeginningNewTransaction.

@Test
public void shouldResetSentOffsetsFlagOnlyWhenBeginningNewTransaction() {
    buildMockProducer(true);
    producer.initTransactions();
    producer.beginTransaction();
    assertFalse(producer.sentOffsets());
    Map<TopicPartition, OffsetAndMetadata> groupCommit = new HashMap<TopicPartition, OffsetAndMetadata>() {

        {
            put(new TopicPartition(topic, 0), new OffsetAndMetadata(42L, null));
        }
    };
    producer.sendOffsetsToTransaction(groupCommit, new ConsumerGroupMetadata("groupId"));
    // commit should not reset "sentOffsets" flag
    producer.commitTransaction();
    assertTrue(producer.sentOffsets());
    producer.beginTransaction();
    assertFalse(producer.sentOffsets());
    producer.sendOffsetsToTransaction(groupCommit, new ConsumerGroupMetadata("groupId"));
    // commit should not reset "sentOffsets" flag
    producer.commitTransaction();
    assertTrue(producer.sentOffsets());
    producer.beginTransaction();
    assertFalse(producer.sentOffsets());
}

Example 37 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class KafkaProducerTest method testMeasureTransactionDurations.

@Test
public void testMeasureTransactionDurations() {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "some.id");
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 10000);
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    Duration tick = Duration.ofSeconds(1);
    Time time = new MockTime(tick.toMillis());
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, singletonMap("topic", 1));
    ProducerMetadata metadata = newMetadata(0, Long.MAX_VALUE);
    MockClient client = new MockClient(time, metadata);
    client.updateMetadata(initialUpdateResponse);
    client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "some.id", NODE));
    client.prepareResponse(initProducerIdResponse(1L, (short) 5, Errors.NONE));
    try (KafkaProducer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time)) {
        producer.initTransactions();
        assertDurationAtLeast(producer, "txn-init-time-ns-total", tick.toNanos());
        client.prepareResponse(addOffsetsToTxnResponse(Errors.NONE));
        client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "some.id", NODE));
        client.prepareResponse(txnOffsetsCommitResponse(Collections.singletonMap(new TopicPartition("topic", 0), Errors.NONE)));
        client.prepareResponse(endTxnResponse(Errors.NONE));
        producer.beginTransaction();
        double beginFirst = getAndAssertDurationAtLeast(producer, "txn-begin-time-ns-total", tick.toNanos());
        producer.sendOffsetsToTransaction(Collections.singletonMap(new TopicPartition("topic", 0), new OffsetAndMetadata(5L)), new ConsumerGroupMetadata("group"));
        double sendOffFirst = getAndAssertDurationAtLeast(producer, "txn-send-offsets-time-ns-total", tick.toNanos());
        producer.commitTransaction();
        double commitFirst = getAndAssertDurationAtLeast(producer, "txn-commit-time-ns-total", tick.toNanos());
        client.prepareResponse(addOffsetsToTxnResponse(Errors.NONE));
        client.prepareResponse(txnOffsetsCommitResponse(Collections.singletonMap(new TopicPartition("topic", 0), Errors.NONE)));
        client.prepareResponse(endTxnResponse(Errors.NONE));
        producer.beginTransaction();
        assertDurationAtLeast(producer, "txn-begin-time-ns-total", beginFirst + tick.toNanos());
        producer.sendOffsetsToTransaction(Collections.singletonMap(new TopicPartition("topic", 0), new OffsetAndMetadata(10L)), new ConsumerGroupMetadata("group"));
        assertDurationAtLeast(producer, "txn-send-offsets-time-ns-total", sendOffFirst + tick.toNanos());
        producer.commitTransaction();
        assertDurationAtLeast(producer, "txn-commit-time-ns-total", commitFirst + tick.toNanos());
    }
}

Example 38 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class KafkaProducerTest method testSendTxnOffsetsWithGroupId.

@Test
public void testSendTxnOffsetsWithGroupId() {
    Map<String, Object> configs = new HashMap<>();
    configs.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "some.id");
    configs.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 10000);
    configs.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9000");
    Time time = new MockTime(1);
    MetadataResponse initialUpdateResponse = RequestTestUtils.metadataUpdateWith(1, singletonMap("topic", 1));
    ProducerMetadata metadata = newMetadata(0, Long.MAX_VALUE);
    MockClient client = new MockClient(time, metadata);
    client.updateMetadata(initialUpdateResponse);
    Node node = metadata.fetch().nodes().get(0);
    client.throttle(node, 5000);
    client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "some.id", NODE));
    client.prepareResponse(initProducerIdResponse(1L, (short) 5, Errors.NONE));
    client.prepareResponse(addOffsetsToTxnResponse(Errors.NONE));
    client.prepareResponse(FindCoordinatorResponse.prepareResponse(Errors.NONE, "some.id", NODE));
    String groupId = "group";
    client.prepareResponse(request -> ((TxnOffsetCommitRequest) request).data().groupId().equals(groupId), txnOffsetsCommitResponse(Collections.singletonMap(new TopicPartition("topic", 0), Errors.NONE)));
    client.prepareResponse(endTxnResponse(Errors.NONE));
    try (Producer<String, String> producer = kafkaProducer(configs, new StringSerializer(), new StringSerializer(), metadata, client, null, time)) {
        producer.initTransactions();
        producer.beginTransaction();
        producer.sendOffsetsToTransaction(Collections.emptyMap(), new ConsumerGroupMetadata(groupId));
        producer.commitTransaction();
    }
}

Example 39 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class TransactionManagerTest method testGroupAuthorizationFailureInTxnOffsetCommit.

@Test
public void testGroupAuthorizationFailureInTxnOffsetCommit() {
    final TopicPartition tp1 = new TopicPartition("foo", 0);
    doInitTransactions();
    transactionManager.beginTransaction();
    TransactionalRequestResult sendOffsetsResult = transactionManager.sendOffsetsToTransaction(singletonMap(tp1, new OffsetAndMetadata(39L)), new ConsumerGroupMetadata(consumerGroupId));
    prepareAddOffsetsToTxnResponse(Errors.NONE, consumerGroupId, producerId, epoch);
    runUntil(() -> !transactionManager.hasPartitionsToAdd());
    prepareFindCoordinatorResponse(Errors.NONE, false, CoordinatorType.GROUP, consumerGroupId);
    prepareTxnOffsetCommitResponse(consumerGroupId, producerId, epoch, singletonMap(tp1, Errors.GROUP_AUTHORIZATION_FAILED));
    runUntil(transactionManager::hasError);
    assertTrue(transactionManager.lastError() instanceof GroupAuthorizationException);
    assertTrue(sendOffsetsResult.isCompleted());
    assertFalse(sendOffsetsResult.isSuccessful());
    assertTrue(sendOffsetsResult.error() instanceof GroupAuthorizationException);
    assertFalse(transactionManager.hasPendingOffsetCommits());
    GroupAuthorizationException exception = (GroupAuthorizationException) sendOffsetsResult.error();
    assertEquals(consumerGroupId, exception.groupId());
    assertAbortableError(GroupAuthorizationException.class);
}

Example 40 with ConsumerGroupMetadata

use of org.apache.kafka.clients.consumer.ConsumerGroupMetadata in project kafka by apache.

the class TransactionManagerTest method testRetriableErrorInTxnOffsetCommit.

private void testRetriableErrorInTxnOffsetCommit(Errors error) {
    doInitTransactions();
    transactionManager.beginTransaction();
    Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
    offsets.put(tp0, new OffsetAndMetadata(1));
    offsets.put(tp1, new OffsetAndMetadata(1));
    TransactionalRequestResult addOffsetsResult = transactionManager.sendOffsetsToTransaction(offsets, new ConsumerGroupMetadata(consumerGroupId));
    prepareAddOffsetsToTxnResponse(Errors.NONE, consumerGroupId, producerId, epoch);
    runUntil(() -> !client.hasPendingResponses());
    // The request should complete only after the TxnOffsetCommit completes.
    assertFalse(addOffsetsResult.isCompleted());
    Map<TopicPartition, Errors> txnOffsetCommitResponse = new HashMap<>();
    txnOffsetCommitResponse.put(tp0, Errors.NONE);
    txnOffsetCommitResponse.put(tp1, error);
    prepareFindCoordinatorResponse(Errors.NONE, false, CoordinatorType.GROUP, consumerGroupId);
    prepareTxnOffsetCommitResponse(consumerGroupId, producerId, epoch, txnOffsetCommitResponse);
    assertNull(transactionManager.coordinator(CoordinatorType.GROUP));
    runUntil(() -> transactionManager.coordinator(CoordinatorType.GROUP) != null);
    assertTrue(transactionManager.hasPendingOffsetCommits());
    // The TxnOffsetCommit failed.
    runUntil(transactionManager::hasPendingOffsetCommits);
    // We should only be done after both RPCs complete successfully.
    assertFalse(addOffsetsResult.isCompleted());
    txnOffsetCommitResponse.put(tp1, Errors.NONE);
    prepareTxnOffsetCommitResponse(consumerGroupId, producerId, epoch, txnOffsetCommitResponse);
    runUntil(addOffsetsResult::isCompleted);
    assertTrue(addOffsetsResult.isSuccessful());
}