use of org.apache.kafka.streams.kstream.KTable in project kafka by apache.
the class StreamsPartitionAssignorTest method shouldGenerateTasksForAllCreatedPartitions.
@Test
public void shouldGenerateTasksForAllCreatedPartitions() {
final StreamsBuilder streamsBuilder = new StreamsBuilder();
// KStream with 3 partitions
final KStream<Object, Object> stream1 = streamsBuilder.stream("topic1").map((KeyValueMapper<Object, Object, KeyValue<Object, Object>>) KeyValue::new);
// KTable with 4 partitions
final KTable<Object, Long> table1 = streamsBuilder.table("topic3").groupBy(KeyValue::new).count();
// joining the stream and the table
// this triggers the enforceCopartitioning() routine in the StreamsPartitionAssignor,
// forcing the stream.map to get repartitioned to a topic with four partitions.
stream1.join(table1, (ValueJoiner<Object, Object, Void>) (value1, value2) -> null);
final String client = "client1";
builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build());
topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(configProps()));
adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(asList(APPLICATION_ID + "-topic3-STATE-STORE-0000000002-changelog", APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-changelog"), asList(4, 4)));
final MockInternalTopicManager mockInternalTopicManager = configureDefault();
subscriptions.put(client, new Subscription(asList("topic1", "topic3"), defaultSubscriptionInfo.encode()));
final Map<String, Assignment> assignment = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
final Map<String, Integer> expectedCreatedInternalTopics = new HashMap<>();
expectedCreatedInternalTopics.put(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 4);
expectedCreatedInternalTopics.put(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-changelog", 4);
expectedCreatedInternalTopics.put(APPLICATION_ID + "-topic3-STATE-STORE-0000000002-changelog", 4);
expectedCreatedInternalTopics.put(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 4);
// check if all internal topics were created as expected
assertThat(mockInternalTopicManager.readyTopics, equalTo(expectedCreatedInternalTopics));
final List<TopicPartition> expectedAssignment = asList(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2), new TopicPartition("topic3", 0), new TopicPartition("topic3", 1), new TopicPartition("topic3", 2), new TopicPartition("topic3", 3), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 0), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 1), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 2), new TopicPartition(APPLICATION_ID + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 3), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 0), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 1), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 2), new TopicPartition(APPLICATION_ID + "-KSTREAM-MAP-0000000001-repartition", 3));
// check if we created a task for all expected topicPartitions.
assertThat(new HashSet<>(assignment.get(client).partitions()), equalTo(new HashSet<>(expectedAssignment)));
}
Aggregations