Search in sources :

Example 56 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testNewSubscription.

@Test
public void testNewSubscription() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    for (int i = 1; i < 5; i++) partitionsPerTopic.put(getTopicName(i, 5), 1);
    for (int i = 0; i < 3; i++) {
        List<String> topics = new ArrayList<>();
        for (int j = i; j <= 3 * i - 2; j++) topics.add(getTopicName(j, 5));
        subscriptions.put(getConsumerName(i, 3), new Subscription(topics));
    }
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    subscriptions.get(getConsumerName(0, 3)).topics().add(getTopicName(1, 5));
    assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(assignor.isSticky());
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Collections.emptyList(java.util.Collections.emptyList) ArrayList(java.util.ArrayList) List(java.util.List) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) Test(org.junit.jupiter.api.Test)

Example 57 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testTwoConsumersOneTopicOnePartition.

@Test
public void testTwoConsumersOneTopicOnePartition() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic, 1);
    subscriptions.put(consumer1, new Subscription(topics(topic)));
    subscriptions.put(consumer2, new Subscription(topics(topic)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    assertTrue(assignor.partitionsTransferringOwnership.isEmpty());
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(isFullyBalanced(assignment));
}
Also used : HashMap(java.util.HashMap) Collections.emptyList(java.util.Collections.emptyList) ArrayList(java.util.ArrayList) List(java.util.List) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) Test(org.junit.jupiter.api.Test)

Example 58 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testLargeAssignmentWithMultipleConsumersLeavingAndRandomSubscription.

@Test
public void testLargeAssignmentWithMultipleConsumersLeavingAndRandomSubscription() {
    Random rand = new Random();
    int topicCount = 40;
    int consumerCount = 200;
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    for (int i = 0; i < topicCount; i++) partitionsPerTopic.put(getTopicName(i, topicCount), rand.nextInt(10) + 1);
    for (int i = 0; i < consumerCount; i++) {
        List<String> topics = new ArrayList<>();
        for (int j = 0; j < rand.nextInt(20); j++) topics.add(getTopicName(rand.nextInt(topicCount), topicCount));
        subscriptions.put(getConsumerName(i, consumerCount), new Subscription(topics));
    }
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    for (int i = 1; i < consumerCount; i++) {
        String consumer = getConsumerName(i, consumerCount);
        subscriptions.put(consumer, buildSubscription(subscriptions.get(consumer).topics(), assignment.get(consumer)));
    }
    for (int i = 0; i < 50; ++i) {
        String c = getConsumerName(rand.nextInt(consumerCount), consumerCount);
        subscriptions.remove(c);
    }
    assignment = assignor.assign(partitionsPerTopic, subscriptions);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(assignor.isSticky());
}
Also used : Random(java.util.Random) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Collections.emptyList(java.util.Collections.emptyList) ArrayList(java.util.ArrayList) List(java.util.List) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) Test(org.junit.jupiter.api.Test)

Example 59 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testTwoConsumersTwoTopicsSixPartitions.

@Test
public void testTwoConsumersTwoTopicsSixPartitions() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic1, 3);
    partitionsPerTopic.put(topic2, 3);
    subscriptions.put(consumer1, new Subscription(topics(topic1, topic2)));
    subscriptions.put(consumer2, new Subscription(topics(topic1, topic2)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    assertEquals(partitions(tp(topic1, 0), tp(topic1, 2), tp(topic2, 1)), assignment.get(consumer1));
    assertEquals(partitions(tp(topic1, 1), tp(topic2, 0), tp(topic2, 2)), assignment.get(consumer2));
    assertTrue(assignor.partitionsTransferringOwnership.isEmpty());
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(isFullyBalanced(assignment));
}
Also used : HashMap(java.util.HashMap) Collections.emptyList(java.util.Collections.emptyList) ArrayList(java.util.ArrayList) List(java.util.List) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) Test(org.junit.jupiter.api.Test)

Example 60 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class AbstractStickyAssignorTest method testMultipleConsumersMixedTopicSubscriptions.

@Test
public void testMultipleConsumersMixedTopicSubscriptions() {
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic1, 3);
    partitionsPerTopic.put(topic2, 2);
    subscriptions.put(consumer1, new Subscription(topics(topic1)));
    subscriptions.put(consumer2, new Subscription(topics(topic1, topic2)));
    subscriptions.put(consumer3, new Subscription(topics(topic1)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, subscriptions);
    assertEquals(partitions(tp(topic1, 0), tp(topic1, 2)), assignment.get(consumer1));
    assertEquals(partitions(tp(topic2, 0), tp(topic2, 1)), assignment.get(consumer2));
    assertEquals(partitions(tp(topic1, 1)), assignment.get(consumer3));
    assertNull(assignor.partitionsTransferringOwnership);
    verifyValidityAndBalance(subscriptions, assignment, partitionsPerTopic);
    assertTrue(isFullyBalanced(assignment));
}
Also used : HashMap(java.util.HashMap) Collections.emptyList(java.util.Collections.emptyList) ArrayList(java.util.ArrayList) List(java.util.List) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) Test(org.junit.jupiter.api.Test)

Aggregations

Subscription (org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription)89 HashMap (java.util.HashMap)50 ArrayList (java.util.ArrayList)49 List (java.util.List)45 Test (org.junit.jupiter.api.Test)44 GroupSubscription (org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription)39 Test (org.junit.Test)33 Collections.emptyList (java.util.Collections.emptyList)28 Assignment (org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment)24 TopicPartition (org.apache.kafka.common.TopicPartition)20 TaskId (org.apache.kafka.streams.processor.TaskId)19 AssignmentInfo (org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo)18 ByteBuffer (java.nio.ByteBuffer)15 MockKeyValueStoreBuilder (org.apache.kafka.test.MockKeyValueStoreBuilder)15 HashSet (java.util.HashSet)14 Map (java.util.Map)13 Cluster (org.apache.kafka.common.Cluster)9 Utils.mkMap (org.apache.kafka.common.utils.Utils.mkMap)9 MockInternalTopicManager (org.apache.kafka.test.MockInternalTopicManager)9 Collections.emptyMap (java.util.Collections.emptyMap)8