Examples with SubscriptionState org.apache.kafka.clients.consumer.internals.SubscriptionState used on opensource projects

Example 1 with SubscriptionState

use of org.apache.kafka.clients.consumer.internals.SubscriptionState in project kafka by apache.

the class KafkaConsumerTest method testMeasureCommittedDuration.

@Test
public void testMeasureCommittedDuration() {
    long offset1 = 10000;
    Time time = new MockTime(Duration.ofSeconds(1).toMillis());
    SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, Collections.singletonMap(topic, 2));
    Node node = metadata.fetch().nodes().get(0);
    KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
    consumer.assign(singletonList(tp0));
    // lookup coordinator
    client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node);
    Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
    // fetch offset for one topic
    client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, offset1), Errors.NONE), coordinator);
    consumer.committed(Collections.singleton(tp0)).get(tp0).offset();
    final Metric metric = consumer.metrics().get(consumer.metrics.metricName("committed-time-ns-total", "consumer-metrics"));
    assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos());
}

Example 2 with SubscriptionState

use of org.apache.kafka.clients.consumer.internals.SubscriptionState in project kafka by apache.

the class KafkaConsumerTest method testResetUsingAutoResetPolicy.

@Test
public void testResetUsingAutoResetPolicy() {
    SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.LATEST);
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, Collections.singletonMap(topic, 1));
    Node node = metadata.fetch().nodes().get(0);
    KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupId, groupInstanceId, false);
    consumer.assign(singletonList(tp0));
    client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node);
    Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
    client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, -1L), Errors.NONE), coordinator);
    client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp0, 50L)));
    consumer.poll(Duration.ZERO);
    assertEquals(50L, consumer.position(tp0));
}

Example 3 with SubscriptionState

use of org.apache.kafka.clients.consumer.internals.SubscriptionState in project kafka by apache.

the class KafkaConsumerTest method testOffsetIsValidAfterSeek.

@Test
public void testOffsetIsValidAfterSeek() {
    SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.LATEST);
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, Collections.singletonMap(topic, 1));
    KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupId, Optional.empty(), false);
    consumer.assign(singletonList(tp0));
    consumer.seek(tp0, 20L);
    consumer.poll(Duration.ZERO);
    assertEquals(subscription.validPosition(tp0).offset, 20L);
}

Example 4 with SubscriptionState

use of org.apache.kafka.clients.consumer.internals.SubscriptionState in project kafka by apache.

the class SubscriptionStateBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    Set<TopicPartition> assignment = new HashSet<>(topicCount * partitionCount);
    IntStream.range(0, topicCount).forEach(topicId -> IntStream.range(0, partitionCount).forEach(partitionId -> assignment.add(new TopicPartition(String.format("topic-%04d", topicId), partitionId))));
    subscriptionState = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
    subscriptionState.assignFromUser(assignment);
    SubscriptionState.FetchPosition position = new SubscriptionState.FetchPosition(0L, Optional.of(0), new Metadata.LeaderAndEpoch(Optional.of(new Node(0, "host", 9092)), Optional.of(10)));
    assignment.forEach(topicPartition -> {
        subscriptionState.seekUnvalidated(topicPartition, position);
        subscriptionState.completeValidation(topicPartition);
    });
}

Example 5 with SubscriptionState

use of org.apache.kafka.clients.consumer.internals.SubscriptionState in project kafka by apache.

the class KafkaConsumerTest method newConsumer.

private KafkaConsumer<String, String> newConsumer(Time time, KafkaClient client, Metadata metadata, PartitionAssignor assignor, int rebalanceTimeoutMs, int sessionTimeoutMs, int heartbeatIntervalMs, boolean autoCommitEnabled, int autoCommitIntervalMs) {
    // create a consumer with mocked time and mocked network
    String clientId = "mock-consumer";
    String groupId = "mock-group";
    String metricGroupPrefix = "consumer";
    long retryBackoffMs = 100;
    long requestTimeoutMs = 30000;
    boolean excludeInternalTopics = true;
    int minBytes = 1;
    int maxBytes = Integer.MAX_VALUE;
    int maxWaitMs = 500;
    int fetchSize = 1024 * 1024;
    int maxPollRecords = Integer.MAX_VALUE;
    boolean checkCrcs = true;
    Deserializer<String> keyDeserializer = new StringDeserializer();
    Deserializer<String> valueDeserializer = new StringDeserializer();
    OffsetResetStrategy autoResetStrategy = OffsetResetStrategy.EARLIEST;
    List<PartitionAssignor> assignors = Arrays.asList(assignor);
    ConsumerInterceptors<String, String> interceptors = null;
    Metrics metrics = new Metrics();
    SubscriptionState subscriptions = new SubscriptionState(autoResetStrategy);
    ConsumerNetworkClient consumerClient = new ConsumerNetworkClient(client, metadata, time, retryBackoffMs, requestTimeoutMs);
    ConsumerCoordinator consumerCoordinator = new ConsumerCoordinator(consumerClient, groupId, rebalanceTimeoutMs, sessionTimeoutMs, heartbeatIntervalMs, assignors, metadata, subscriptions, metrics, metricGroupPrefix, time, retryBackoffMs, autoCommitEnabled, autoCommitIntervalMs, interceptors, excludeInternalTopics);
    Fetcher<String, String> fetcher = new Fetcher<>(consumerClient, minBytes, maxBytes, maxWaitMs, fetchSize, maxPollRecords, checkCrcs, keyDeserializer, valueDeserializer, metadata, subscriptions, metrics, metricGroupPrefix, time, retryBackoffMs);
    return new KafkaConsumer<>(clientId, consumerCoordinator, keyDeserializer, valueDeserializer, fetcher, interceptors, time, consumerClient, metrics, subscriptions, metadata, retryBackoffMs, requestTimeoutMs);
}