Examples with Subscription org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription used on opensource projects

Example 56 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testNewSubscription.

@Test
public void testNewSubscription() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    for (int i = 1; i < 5; i++) partitionsPerTopic.put(getTopicName(i, 5), 1);
    for (int i = 0; i < 3; i++) {
        List<String> topics = new ArrayList<>();
        for (int j = i; j <= 3 * i - 2; j++) topics.add(getTopicName(j, 5));
        subscriptions.put(getConsumerName(i, 3), new Subscription(topics));
    }
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    subscriptions.get(getConsumerName(0, 3)).topics().add(getTopicName(1, 5));
    assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(assignor.isSticky());
}

Example 57 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testTwoConsumersOneTopicOnePartition.

@Test
public void testTwoConsumersOneTopicOnePartition() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic, 1);
    subscriptions.put(consumer1, new Subscription(topics(topic)));
    subscriptions.put(consumer2, new Subscription(topics(topic)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    assertTrue(assignor.partitionsTransferringOwnership.isEmpty());
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(isFullyBalanced(assignment));
}

Example 58 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testLargeAssignmentWithMultipleConsumersLeavingAndRandomSubscription.

@Test
public void testLargeAssignmentWithMultipleConsumersLeavingAndRandomSubscription() {
    Random rand = new Random();
    int topicCount = 40;
    int consumerCount = 200;
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    for (int i = 0; i < topicCount; i++) partitionsPerTopic.put(getTopicName(i, topicCount), rand.nextInt(10) + 1);
    for (int i = 0; i < consumerCount; i++) {
        List<String> topics = new ArrayList<>();
        for (int j = 0; j < rand.nextInt(20); j++) topics.add(getTopicName(rand.nextInt(topicCount), topicCount));
        subscriptions.put(getConsumerName(i, consumerCount), new Subscription(topics));
    }
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    for (int i = 1; i < consumerCount; i++) {
        String consumer = getConsumerName(i, consumerCount);
        subscriptions.put(consumer, buildSubscription(subscriptions.get(consumer).topics(), assignment.get(consumer)));
    }
    for (int i = 0; i < 50; ++i) {
        String c = getConsumerName(rand.nextInt(consumerCount), consumerCount);
        subscriptions.remove(c);
    }
    assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(assignor.isSticky());
}

Example 59 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testTwoConsumersTwoTopicsSixPartitions.

@Test
public void testTwoConsumersTwoTopicsSixPartitions() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic1, 3);
    partitionsPerTopic.put(topic2, 3);
    subscriptions.put(consumer1, new Subscription(topics(topic1, topic2)));
    subscriptions.put(consumer2, new Subscription(topics(topic1, topic2)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    assertEquals(partitions(tp(topic1, 0), tp(topic1, 2), tp(topic2, 1)), assignment.get(consumer1));
    assertEquals(partitions(tp(topic1, 1), tp(topic2, 0), tp(topic2, 2)), assignment.get(consumer2));
    assertTrue(assignor.partitionsTransferringOwnership.isEmpty());
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(isFullyBalanced(assignment));
}

Example 60 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testMultipleConsumersMixedTopicSubscriptions.

@Test
public void testMultipleConsumersMixedTopicSubscriptions() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic1, 3);
    partitionsPerTopic.put(topic2, 2);
    subscriptions.put(consumer1, new Subscription(topics(topic1)));
    subscriptions.put(consumer2, new Subscription(topics(topic1, topic2)));
    subscriptions.put(consumer3, new Subscription(topics(topic1)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    assertEquals(partitions(tp(topic1, 0), tp(topic1, 2)), assignment.get(consumer1));
    assertEquals(partitions(tp(topic2, 0), tp(topic2, 1)), assignment.get(consumer2));
    assertEquals(partitions(tp(topic1, 1)), assignment.get(consumer3));
    assertNull(assignor.partitionsTransferringOwnership);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(isFullyBalanced(assignment));
}