Examples with Subscription org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription used on opensource projects

Example 1 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class CooperativeStickyAssignorTest method verifyValidityAndBalance.

/**
 * The cooperative assignor must do some additional work and verification of some assignments relative to the eager
 * assignor, since it may or may not need to trigger a second follow-up rebalance.
 * <p>
 * In addition to the validity requirements described in
 * {@link org.apache.kafka.clients.consumer.internals.AbstractStickyAssignorTest#verifyValidityAndBalance(Map, Map, Map)},
 * we must verify that no partition is being revoked and reassigned during the same rebalance. This means the initial
 * assignment may be unbalanced, so if we do detect partitions being revoked we should trigger a second "rebalance"
 * to get the final assignment and then verify that it is both valid and balanced.
 */
@Override
public void verifyValidityAndBalance(Map<String, Subscription> subscriptions, Map<String, List<TopicPartition>> assignments, Map<String, Integer> partitionsPerTopic) {
    int rebalances = 0;
    // partitions are being revoked, we must go through another assignment to get the final state
    while (verifyCooperativeValidity(subscriptions, assignments)) {
        // update the subscriptions with the now owned partitions
        for (Map.Entry<String, List<TopicPartition>> entry : assignments.entrySet()) {
            String consumer = entry.getKey();
            Subscription oldSubscription = subscriptions.get(consumer);
            subscriptions.put(consumer, buildSubscription(oldSubscription.topics(), entry.getValue()));
        }
        assignments.clear();
        assignments.putAll(assignor.assign(partitionsPerTopic, subscriptions));
        ++rebalances;
        assertTrue(rebalances <= 4);
    }
    // Check the validity and balance of the final assignment
    super.verifyValidityAndBalance(subscriptions, assignments, partitionsPerTopic);
}

Example 2 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class CooperativeStickyAssignorTest method testDecodeGeneration.

@Test
public void testDecodeGeneration() {
    Subscription subscription = new Subscription(topics(topic));
    assertFalse(((CooperativeStickyAssignor) assignor).memberData(subscription).generation.isPresent());
}

Example 3 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class RangeAssignorTest method testTwoConsumersOneTopicOnePartition.

@Test
public void testTwoConsumersOneTopicOnePartition() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic1, 1);
    Map<String, Subscription> consumers = new HashMap<>();
    consumers.put(consumer1, new Subscription(topics(topic1)));
    consumers.put(consumer2, new Subscription(topics(topic1)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, consumers);
    assertAssignment(partitions(tp(topic1, 0)), assignment.get(consumer1));
    assertAssignment(Collections.emptyList(), assignment.get(consumer2));
}

Example 4 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class RangeAssignorTest method testTwoConsumersTwoTopicsSixPartitions.

@Test
public void testTwoConsumersTwoTopicsSixPartitions() {
    String topic1 = "topic1";
    String topic2 = "topic2";
    String consumer1 = "consumer1";
    String consumer2 = "consumer2";
    Map<String, Integer> partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
    Map<String, Subscription> consumers = new HashMap<>();
    consumers.put(consumer1, new Subscription(topics(topic1, topic2)));
    consumers.put(consumer2, new Subscription(topics(topic1, topic2)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, consumers);
    assertAssignment(partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0), tp(topic2, 1)), assignment.get(consumer1));
    assertAssignment(partitions(tp(topic1, 2), tp(topic2, 2)), assignment.get(consumer2));
}

Example 5 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class RangeAssignorTest method testTwoStaticConsumersTwoTopicsSixPartitions.

@Test
public void testTwoStaticConsumersTwoTopicsSixPartitions() {
    // although consumer high has a higher rank than consumer low, the comparison happens on
    // instance id level.
    String consumerIdLow = "consumer-b";
    String consumerIdHigh = "consumer-a";
    Map<String, Integer> partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
    Map<String, Subscription> consumers = new HashMap<>();
    Subscription consumerLowSubscription = new Subscription(topics(topic1, topic2), null, Collections.emptyList());
    consumerLowSubscription.setGroupInstanceId(Optional.of(instance1));
    consumers.put(consumerIdLow, consumerLowSubscription);
    Subscription consumerHighSubscription = new Subscription(topics(topic1, topic2), null, Collections.emptyList());
    consumerHighSubscription.setGroupInstanceId(Optional.of(instance2));
    consumers.put(consumerIdHigh, consumerHighSubscription);
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, consumers);
    assertAssignment(partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0), tp(topic2, 1)), assignment.get(consumerIdLow));
    assertAssignment(partitions(tp(topic1, 2), tp(topic2, 2)), assignment.get(consumerIdHigh));
}