Examples with ConsumerMetadata org.apache.kafka.clients.consumer.internals.ConsumerMetadata used on opensource projects

Example 36 with ConsumerMetadata

use of org.apache.kafka.clients.consumer.internals.ConsumerMetadata in project kafka by apache.

the class KafkaConsumerTest method testSubscriptionChangesWithAutoCommitDisabled.

/**
 * Verify that when a consumer changes its topic subscription its assigned partitions
 * do not immediately change, and the consumed offsets of its to-be-revoked partitions
 * are not committed (when auto-commit is disabled).
 * Upon unsubscribing from subscribed topics, the assigned partitions immediately
 * change but if auto-commit is disabled the consumer offsets are not committed.
 */
@Test
public void testSubscriptionChangesWithAutoCommitDisabled() {
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    Map<String, Integer> tpCounts = new HashMap<>();
    tpCounts.put(topic, 1);
    tpCounts.put(topic2, 1);
    initMetadata(client, tpCounts);
    Node node = metadata.fetch().nodes().get(0);
    ConsumerPartitionAssignor assignor = new RangeAssignor();
    KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
    initializeSubscriptionWithSingleTopic(consumer, getConsumerRebalanceListener(consumer));
    // mock rebalance responses
    prepareRebalance(client, node, assignor, singletonList(tp0), null);
    consumer.updateAssignmentMetadataIfNeeded(time.timer(Long.MAX_VALUE));
    consumer.poll(Duration.ZERO);
    // verify that subscription is still the same, and now assignment has caught up
    assertEquals(singleton(topic), consumer.subscription());
    assertEquals(singleton(tp0), consumer.assignment());
    consumer.poll(Duration.ZERO);
    // subscription change
    consumer.subscribe(singleton(topic2), getConsumerRebalanceListener(consumer));
    // verify that subscription has changed but assignment is still unchanged
    assertEquals(singleton(topic2), consumer.subscription());
    assertEquals(singleton(tp0), consumer.assignment());
    // the auto commit is disabled, so no offset commit request should be sent
    for (ClientRequest req : client.requests()) assertNotSame(ApiKeys.OFFSET_COMMIT, req.requestBuilder().apiKey());
    // subscription change
    consumer.unsubscribe();
    // verify that subscription and assignment are both updated
    assertEquals(Collections.emptySet(), consumer.subscription());
    assertEquals(Collections.emptySet(), consumer.assignment());
    // the auto commit is disabled, so no offset commit request should be sent
    for (ClientRequest req : client.requests()) assertNotSame(ApiKeys.OFFSET_COMMIT, req.requestBuilder().apiKey());
    client.requests().clear();
    consumer.close();
}

Example 37 with ConsumerMetadata

use of org.apache.kafka.clients.consumer.internals.ConsumerMetadata in project kafka by apache.

the class KafkaConsumerTest method testEnforceRebalanceTriggersRebalanceOnNextPoll.

@Test
public void testEnforceRebalanceTriggersRebalanceOnNextPoll() {
    Time time = new MockTime(1L);
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
    MockRebalanceListener countingRebalanceListener = new MockRebalanceListener();
    initMetadata(client, Utils.mkMap(Utils.mkEntry(topic, 1), Utils.mkEntry(topic2, 1), Utils.mkEntry(topic3, 1)));
    consumer.subscribe(Arrays.asList(topic, topic2), countingRebalanceListener);
    Node node = metadata.fetch().nodes().get(0);
    prepareRebalance(client, node, assignor, Arrays.asList(tp0, t2p0), null);
    // a first rebalance to get the assignment, we need two poll calls since we need two round trips to finish join / sync-group
    consumer.poll(Duration.ZERO);
    consumer.poll(Duration.ZERO);
    // onPartitionsRevoked is not invoked when first joining the group
    assertEquals(countingRebalanceListener.revokedCount, 0);
    assertEquals(countingRebalanceListener.assignedCount, 1);
    consumer.enforceRebalance();
    // the next poll should trigger a rebalance
    consumer.poll(Duration.ZERO);
    assertEquals(countingRebalanceListener.revokedCount, 1);
}

Example 38 with ConsumerMetadata

use of org.apache.kafka.clients.consumer.internals.ConsumerMetadata in project kafka by apache.

the class KafkaConsumerTest method consumerForCheckingTimeoutException.

private KafkaConsumer<String, String> consumerForCheckingTimeoutException() {
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, singletonMap(topic, 1));
    ConsumerPartitionAssignor assignor = new RangeAssignor();
    final KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
    for (int i = 0; i < 10; i++) {
        client.prepareResponse(request -> {
            time.sleep(defaultApiTimeoutMs / 10);
            return request instanceof ListOffsetsRequest;
        }, listOffsetsResponse(Collections.emptyMap(), Collections.singletonMap(tp0, Errors.UNKNOWN_TOPIC_OR_PARTITION)));
    }
    return consumer;
}

Example 39 with ConsumerMetadata

use of org.apache.kafka.clients.consumer.internals.ConsumerMetadata in project kafka by apache.

the class KafkaConsumerTest method consumerWithPendingAuthenticationError.

private KafkaConsumer<String, String> consumerWithPendingAuthenticationError(final Time time) {
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, singletonMap(topic, 1));
    Node node = metadata.fetch().nodes().get(0);
    ConsumerPartitionAssignor assignor = new RangeAssignor();
    client.createPendingAuthenticationError(node, 0);
    return newConsumer(time, client, subscription, metadata, assignor, false, groupInstanceId);
}

Example 40 with ConsumerMetadata

use of org.apache.kafka.clients.consumer.internals.ConsumerMetadata in project kafka by apache.

the class KafkaConsumerTest method testPollTimeMetrics.

@Test
public void testPollTimeMetrics() {
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, Collections.singletonMap(topic, 1));
    KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
    consumer.subscribe(singletonList(topic));
    // MetricName objects to check
    Metrics metrics = consumer.metrics;
    MetricName lastPollSecondsAgoName = metrics.metricName("last-poll-seconds-ago", "consumer-metrics");
    MetricName timeBetweenPollAvgName = metrics.metricName("time-between-poll-avg", "consumer-metrics");
    MetricName timeBetweenPollMaxName = metrics.metricName("time-between-poll-max", "consumer-metrics");
    // Test default values
    assertEquals(-1.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
    assertEquals(Double.NaN, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
    assertEquals(Double.NaN, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
    // Call first poll
    consumer.poll(Duration.ZERO);
    assertEquals(0.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
    assertEquals(0.0d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
    assertEquals(0.0d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
    // Advance time by 5,000 (total time = 5,000)
    time.sleep(5 * 1000L);
    assertEquals(5.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
    // Call second poll
    consumer.poll(Duration.ZERO);
    assertEquals(2.5 * 1000d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
    assertEquals(5 * 1000d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
    // Advance time by 10,000 (total time = 15,000)
    time.sleep(10 * 1000L);
    assertEquals(10.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
    // Call third poll
    consumer.poll(Duration.ZERO);
    assertEquals(5 * 1000d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
    assertEquals(10 * 1000d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
    // Advance time by 5,000 (total time = 20,000)
    time.sleep(5 * 1000L);
    assertEquals(5.0d, consumer.metrics().get(lastPollSecondsAgoName).metricValue());
    // Call fourth poll
    consumer.poll(Duration.ZERO);
    assertEquals(5 * 1000d, consumer.metrics().get(timeBetweenPollAvgName).metricValue());
    assertEquals(10 * 1000d, consumer.metrics().get(timeBetweenPollMaxName).metricValue());
}