Search in sources :

Example 41 with AssignmentInfo

use of org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method testAssignWithNewTasks.

@Test
public void testAssignWithNewTasks() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addSource(null, "source2", null, null, null, "topic2");
    builder.addSource(null, "source3", null, null, null, "topic3");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source1", "source2", "source3");
    final List<String> topics = asList("topic1", "topic2", "topic3");
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3);
    // assuming that previous tasks do not have topic3
    final Set<TaskId> prevTasks10 = mkSet(TASK_0_0);
    final Set<TaskId> prevTasks11 = mkSet(TASK_0_1);
    final Set<TaskId> prevTasks20 = mkSet(TASK_0_2);
    createMockTaskManager(prevTasks10, EMPTY_TASKS);
    configureDefaultPartitionAssignor();
    subscriptions.put("consumer10", new Subscription(topics, getInfo(UUID_1, prevTasks10, EMPTY_TASKS).encode()));
    subscriptions.put("consumer11", new Subscription(topics, getInfo(UUID_1, prevTasks11, EMPTY_TASKS).encode()));
    subscriptions.put("consumer20", new Subscription(topics, getInfo(UUID_2, prevTasks20, EMPTY_TASKS).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    // check assigned partitions: since there is no previous task for topic 3 it will be assigned randomly so we cannot check exact match
    // also note that previously assigned partitions / tasks may not stay on the previous host since we may assign the new task first and
    // then later ones will be re-assigned to other hosts due to load balancing
    AssignmentInfo info = AssignmentInfo.decode(assignments.get("consumer10").userData());
    final Set<TaskId> allActiveTasks = new HashSet<>(info.activeTasks());
    final Set<TopicPartition> allPartitions = new HashSet<>(assignments.get("consumer10").partitions());
    info = AssignmentInfo.decode(assignments.get("consumer11").userData());
    allActiveTasks.addAll(info.activeTasks());
    allPartitions.addAll(assignments.get("consumer11").partitions());
    info = AssignmentInfo.decode(assignments.get("consumer20").userData());
    allActiveTasks.addAll(info.activeTasks());
    allPartitions.addAll(assignments.get("consumer20").partitions());
    assertEquals(allTasks, allActiveTasks);
    assertEquals(mkSet(t1p0, t1p1, t1p2, t2p0, t2p1, t2p2, t3p0, t3p1, t3p2, t3p3), allPartitions);
}
Also used : TaskId(org.apache.kafka.streams.processor.TaskId) Assignment(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment) AssignmentInfo(org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo) TopicPartition(org.apache.kafka.common.TopicPartition) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 42 with AssignmentInfo

use of org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldNotAddStandbyTaskPartitionsToPartitionsForHost.

@Test
public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() {
    final Map<String, Object> props = configProps();
    props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
    props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, USER_END_POINT);
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    streamsBuilder.stream("topic1").groupByKey().count();
    builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build());
    topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(props));
    createDefaultMockTaskManager();
    adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(singletonList(APPLICATION_ID + "-KSTREAM-AGGREGATE-STATE-STORE-0000000001-changelog"), singletonList(3)));
    configurePartitionAssignorWith(props);
    subscriptions.put("consumer1", new Subscription(Collections.singletonList("topic1"), getInfo(UUID_1, EMPTY_TASKS, EMPTY_TASKS, USER_END_POINT).encode()));
    subscriptions.put("consumer2", new Subscription(Collections.singletonList("topic1"), getInfo(UUID_2, EMPTY_TASKS, EMPTY_TASKS, OTHER_END_POINT).encode()));
    final Set<TopicPartition> allPartitions = mkSet(t1p0, t1p1, t1p2);
    final Map<String, Assignment> assign = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    final Assignment consumer1Assignment = assign.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
    final Set<TopicPartition> consumer1ActivePartitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
    final Set<TopicPartition> consumer2ActivePartitions = assignmentInfo.partitionsByHost().get(new HostInfo("other", 9090));
    final Set<TopicPartition> consumer1StandbyPartitions = assignmentInfo.standbyPartitionByHost().get(new HostInfo("localhost", 8080));
    final Set<TopicPartition> consumer2StandbyPartitions = assignmentInfo.standbyPartitionByHost().get(new HostInfo("other", 9090));
    final HashSet<TopicPartition> allAssignedPartitions = new HashSet<>(consumer1ActivePartitions);
    allAssignedPartitions.addAll(consumer2ActivePartitions);
    assertThat(consumer1ActivePartitions, not(allPartitions));
    assertThat(consumer2ActivePartitions, not(allPartitions));
    assertThat(consumer1ActivePartitions, equalTo(consumer2StandbyPartitions));
    assertThat(consumer2ActivePartitions, equalTo(consumer1StandbyPartitions));
    assertThat(allAssignedPartitions, equalTo(allPartitions));
}
Also used : InternalStreamsBuilder(org.apache.kafka.streams.kstream.internals.InternalStreamsBuilder) StreamsBuilder(org.apache.kafka.streams.StreamsBuilder) Assignment(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment) AssignmentInfo(org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo) TopicPartition(org.apache.kafka.common.TopicPartition) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) HostInfo(org.apache.kafka.streams.state.HostInfo) StreamsConfig(org.apache.kafka.streams.StreamsConfig) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 43 with AssignmentInfo

use of org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldMapUserEndPointToTopicPartitions.

@Test
public void shouldMapUserEndPointToTopicPartitions() {
    builder.addSource(null, "source", null, null, null, "topic1");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source");
    builder.addSink("sink", "output", null, null, null, "processor");
    final List<String> topics = Collections.singletonList("topic1");
    createDefaultMockTaskManager();
    configurePartitionAssignorWith(Collections.singletonMap(StreamsConfig.APPLICATION_SERVER_CONFIG, USER_END_POINT));
    subscriptions.put("consumer1", new Subscription(topics, getInfo(UUID_1, EMPTY_TASKS, EMPTY_TASKS, USER_END_POINT).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    final Assignment consumerAssignment = assignments.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumerAssignment.userData());
    final Set<TopicPartition> topicPartitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
    assertEquals(mkSet(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2)), topicPartitions);
}
Also used : Assignment(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment) AssignmentInfo(org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo) TopicPartition(org.apache.kafka.common.TopicPartition) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) HostInfo(org.apache.kafka.streams.state.HostInfo) Test(org.junit.Test)

Example 44 with AssignmentInfo

use of org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method testAssignWithStandbyReplicas.

@Test
public void testAssignWithStandbyReplicas() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addSource(null, "source2", null, null, null, "topic2");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source1", "source2");
    builder.addStateStore(new MockKeyValueStoreBuilder("store1", false), "processor");
    final List<String> topics = asList("topic1", "topic2");
    final Set<TopicPartition> allTopicPartitions = topics.stream().map(topic -> asList(new TopicPartition(topic, 0), new TopicPartition(topic, 1), new TopicPartition(topic, 2))).flatMap(Collection::stream).collect(Collectors.toSet());
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2);
    final Set<TaskId> prevTasks00 = mkSet(TASK_0_0);
    final Set<TaskId> prevTasks01 = mkSet(TASK_0_1);
    final Set<TaskId> prevTasks02 = mkSet(TASK_0_2);
    final Set<TaskId> standbyTasks00 = mkSet(TASK_0_0);
    final Set<TaskId> standbyTasks01 = mkSet(TASK_0_1);
    final Set<TaskId> standbyTasks02 = mkSet(TASK_0_2);
    createMockTaskManager(prevTasks00, standbyTasks01);
    adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(singletonList(APPLICATION_ID + "-store1-changelog"), singletonList(3)));
    configurePartitionAssignorWith(Collections.singletonMap(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1));
    subscriptions.put("consumer10", new Subscription(topics, getInfo(UUID_1, prevTasks00, EMPTY_TASKS, USER_END_POINT).encode()));
    subscriptions.put("consumer11", new Subscription(topics, getInfo(UUID_1, prevTasks01, standbyTasks02, USER_END_POINT).encode()));
    subscriptions.put("consumer20", new Subscription(topics, getInfo(UUID_2, prevTasks02, standbyTasks00, OTHER_END_POINT).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    // the first consumer
    final AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10"));
    final Set<TaskId> allActiveTasks = new HashSet<>(info10.activeTasks());
    final Set<TaskId> allStandbyTasks = new HashSet<>(info10.standbyTasks().keySet());
    // the second consumer
    final AssignmentInfo info11 = checkAssignment(allTopics, assignments.get("consumer11"));
    allActiveTasks.addAll(info11.activeTasks());
    allStandbyTasks.addAll(info11.standbyTasks().keySet());
    assertNotEquals("same processId has same set of standby tasks", info11.standbyTasks().keySet(), info10.standbyTasks().keySet());
    // check active tasks assigned to the first client
    assertEquals(mkSet(TASK_0_0, TASK_0_1), new HashSet<>(allActiveTasks));
    assertEquals(mkSet(TASK_0_2), new HashSet<>(allStandbyTasks));
    // the third consumer
    final AssignmentInfo info20 = checkAssignment(allTopics, assignments.get("consumer20"));
    allActiveTasks.addAll(info20.activeTasks());
    allStandbyTasks.addAll(info20.standbyTasks().keySet());
    // all task ids are in the active tasks and also in the standby tasks
    assertEquals(3, allActiveTasks.size());
    assertEquals(allTasks, allActiveTasks);
    assertEquals(3, allStandbyTasks.size());
    assertEquals(allTasks, allStandbyTasks);
    // Check host partition assignments
    final Map<HostInfo, Set<TopicPartition>> partitionsByHost = info10.partitionsByHost();
    assertEquals(2, partitionsByHost.size());
    assertEquals(allTopicPartitions, partitionsByHost.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()));
    final Map<HostInfo, Set<TopicPartition>> standbyPartitionsByHost = info10.standbyPartitionByHost();
    assertEquals(2, standbyPartitionsByHost.size());
    assertEquals(allTopicPartitions, standbyPartitionsByHost.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()));
    for (final HostInfo hostInfo : partitionsByHost.keySet()) {
        assertTrue(Collections.disjoint(partitionsByHost.get(hostInfo), standbyPartitionsByHost.get(hostInfo)));
    }
    // All consumers got the same host info
    assertEquals(partitionsByHost, info11.partitionsByHost());
    assertEquals(partitionsByHost, info20.partitionsByHost());
    assertEquals(standbyPartitionsByHost, info11.standbyPartitionByHost());
    assertEquals(standbyPartitionsByHost, info20.standbyPartitionByHost());
}
Also used : TaskId(org.apache.kafka.streams.processor.TaskId) Utils.mkSet(org.apache.kafka.common.utils.Utils.mkSet) Set(java.util.Set) Utils.mkSortedSet(org.apache.kafka.common.utils.Utils.mkSortedSet) SortedSet(java.util.SortedSet) HashSet(java.util.HashSet) Collections.emptySet(java.util.Collections.emptySet) Assignment(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment) AssignmentInfo(org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo) TopicPartition(org.apache.kafka.common.TopicPartition) Collection(java.util.Collection) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) HostInfo(org.apache.kafka.streams.state.HostInfo) MockKeyValueStoreBuilder(org.apache.kafka.test.MockKeyValueStoreBuilder) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 45 with AssignmentInfo

use of org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method testAssignWithStandbyReplicasAndStatelessTasks.

@Test
public void testAssignWithStandbyReplicasAndStatelessTasks() {
    builder.addSource(null, "source1", null, null, null, "topic1", "topic2");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source1");
    final List<String> topics = asList("topic1", "topic2");
    createMockTaskManager(mkSet(TASK_0_0), emptySet());
    configurePartitionAssignorWith(Collections.singletonMap(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1));
    subscriptions.put("consumer10", new Subscription(topics, getInfo(UUID_1, mkSet(TASK_0_0), emptySet()).encode()));
    subscriptions.put("consumer20", new Subscription(topics, getInfo(UUID_2, mkSet(TASK_0_2), emptySet()).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    final AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10"));
    assertTrue(info10.standbyTasks().isEmpty());
    final AssignmentInfo info20 = checkAssignment(allTopics, assignments.get("consumer20"));
    assertTrue(info20.standbyTasks().isEmpty());
}
Also used : Assignment(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment) AssignmentInfo(org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) GroupSubscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription) Subscription(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription) Test(org.junit.Test)

Aggregations

AssignmentInfo (org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo)46 TaskId (org.apache.kafka.streams.processor.TaskId)41 HashSet (java.util.HashSet)38 Test (org.junit.Test)35 HashMap (java.util.HashMap)29 TopicPartition (org.apache.kafka.common.TopicPartition)24 UUID (java.util.UUID)21 PartitionAssignor (org.apache.kafka.clients.consumer.internals.PartitionAssignor)21 SubscriptionInfo (org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo)20 HostInfo (org.apache.kafka.streams.state.HostInfo)17 Set (java.util.Set)16 MockInternalTopicManager (org.apache.kafka.test.MockInternalTopicManager)16 MockProcessorSupplier (org.apache.kafka.test.MockProcessorSupplier)16 Assignment (org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment)15 GroupSubscription (org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.GroupSubscription)14 Subscription (org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription)14 ArrayList (java.util.ArrayList)12 Cluster (org.apache.kafka.common.Cluster)12 PartitionInfo (org.apache.kafka.common.PartitionInfo)10 StreamsBuilderTest (org.apache.kafka.streams.StreamsBuilderTest)10