use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.
the class StreamsMetadataStateTest method before.
@Before
public void before() throws Exception {
builder = new StreamsBuilder();
final KStream<Object, Object> one = builder.stream("topic-one");
one.groupByKey().count("table-one");
final KStream<Object, Object> two = builder.stream("topic-two");
two.groupByKey().count("table-two");
builder.stream("topic-three").groupByKey().count("table-three");
one.merge(two).groupByKey().count("merged-table");
builder.stream("topic-four").mapValues(new ValueMapper<Object, Object>() {
@Override
public Object apply(final Object value) {
return value;
}
});
builder.globalTable("global-topic", Consumed.with(null, null), Materialized.<Object, Object, KeyValueStore<Bytes, byte[]>>as(globalTable));
StreamsBuilderTest.internalTopologyBuilder(builder).setApplicationId("appId");
topic1P0 = new TopicPartition("topic-one", 0);
topic1P1 = new TopicPartition("topic-one", 1);
topic2P0 = new TopicPartition("topic-two", 0);
topic2P1 = new TopicPartition("topic-two", 1);
topic3P0 = new TopicPartition("topic-three", 0);
topic4P0 = new TopicPartition("topic-four", 0);
hostOne = new HostInfo("host-one", 8080);
hostTwo = new HostInfo("host-two", 9090);
hostThree = new HostInfo("host-three", 7070);
hostToPartitions = new HashMap<>();
hostToPartitions.put(hostOne, Utils.mkSet(topic1P0, topic2P1, topic4P0));
hostToPartitions.put(hostTwo, Utils.mkSet(topic2P0, topic1P1));
hostToPartitions.put(hostThree, Collections.singleton(topic3P0));
partitionInfos = Arrays.asList(new PartitionInfo("topic-one", 0, null, null, null), new PartitionInfo("topic-one", 1, null, null, null), new PartitionInfo("topic-two", 0, null, null, null), new PartitionInfo("topic-two", 1, null, null, null), new PartitionInfo("topic-three", 0, null, null, null), new PartitionInfo("topic-four", 0, null, null, null));
cluster = new Cluster(null, Collections.<Node>emptyList(), partitionInfos, Collections.<String>emptySet(), Collections.<String>emptySet());
discovery = new StreamsMetadataState(StreamsBuilderTest.internalTopologyBuilder(builder), hostOne);
discovery.onChange(hostToPartitions, cluster);
partitioner = new StreamPartitioner<String, Object>() {
@Override
public Integer partition(final String key, final Object value, final int numPartitions) {
return 1;
}
};
}
use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.
the class StreamsPartitionAssignorTest method shouldNotAddStandbyTaskPartitionsToPartitionsForHost.
@Test
public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() throws Exception {
final StreamsBuilder builder = new StreamsBuilder();
final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder);
internalTopologyBuilder.setApplicationId(applicationId);
builder.stream("topic1").groupByKey().count();
final UUID uuid = UUID.randomUUID();
mockTaskManager(Collections.<TaskId>emptySet(), Collections.<TaskId>emptySet(), uuid, internalTopologyBuilder);
Map<String, Object> props = new HashMap<>();
props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, userEndPoint);
configurePartitionAssignor(props);
partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
final Set<TaskId> emptyTasks = Collections.emptySet();
subscriptions.put("consumer1", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
subscriptions.put("consumer2", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(UUID.randomUUID(), emptyTasks, emptyTasks, "other:9090").encode()));
final Set<TopicPartition> allPartitions = Utils.mkSet(t1p0, t1p1, t1p2);
final Map<String, PartitionAssignor.Assignment> assign = partitionAssignor.assign(metadata, subscriptions);
final PartitionAssignor.Assignment consumer1Assignment = assign.get("consumer1");
final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
final Set<TopicPartition> consumer1partitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
final Set<TopicPartition> consumer2Partitions = assignmentInfo.partitionsByHost().get(new HostInfo("other", 9090));
final HashSet<TopicPartition> allAssignedPartitions = new HashSet<>(consumer1partitions);
allAssignedPartitions.addAll(consumer2Partitions);
assertThat(consumer1partitions, not(allPartitions));
assertThat(consumer2Partitions, not(allPartitions));
assertThat(allAssignedPartitions, equalTo(allPartitions));
}
use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.
the class StreamsPartitionAssignorTest method shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks.
@Test
public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks() throws Exception {
final StreamsBuilder builder = new StreamsBuilder();
final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder);
internalTopologyBuilder.setApplicationId(applicationId);
KStream<Object, Object> stream1 = builder.stream("topic1").selectKey(new KeyValueMapper<Object, Object, Object>() {
@Override
public Object apply(final Object key, final Object value) {
return null;
}
}).groupByKey().count(Materialized.<Object, Long, KeyValueStore<Bytes, byte[]>>as("count")).toStream().map(new KeyValueMapper<Object, Long, KeyValue<Object, Object>>() {
@Override
public KeyValue<Object, Object> apply(final Object key, final Long value) {
return null;
}
});
builder.stream("unknownTopic").selectKey(new KeyValueMapper<Object, Object, Object>() {
@Override
public Object apply(final Object key, final Object value) {
return null;
}
}).join(stream1, new ValueJoiner() {
@Override
public Object apply(final Object value1, final Object value2) {
return null;
}
}, JoinWindows.of(0));
final UUID uuid = UUID.randomUUID();
final String client = "client1";
mockTaskManager(Collections.<TaskId>emptySet(), Collections.<TaskId>emptySet(), UUID.randomUUID(), internalTopologyBuilder);
configurePartitionAssignor(Collections.<String, Object>emptyMap());
final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer);
partitionAssignor.setInternalTopicManager(mockInternalTopicManager);
final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
final Set<TaskId> emptyTasks = Collections.emptySet();
subscriptions.put(client, new PartitionAssignor.Subscription(Collections.singletonList("unknownTopic"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
final Map<String, PartitionAssignor.Assignment> assignment = partitionAssignor.assign(metadata, subscriptions);
final Map<String, Integer> expectedCreatedInternalTopics = new HashMap<>();
expectedCreatedInternalTopics.put(applicationId + "-count-repartition", 3);
expectedCreatedInternalTopics.put(applicationId + "-count-changelog", 3);
assertThat(mockInternalTopicManager.readyTopics, equalTo(expectedCreatedInternalTopics));
final List<TopicPartition> expectedAssignment = Arrays.asList(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2), new TopicPartition(applicationId + "-count-repartition", 0), new TopicPartition(applicationId + "-count-repartition", 1), new TopicPartition(applicationId + "-count-repartition", 2));
assertThat(new HashSet<>(assignment.get(client).partitions()), equalTo(new HashSet<>(expectedAssignment)));
}
use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.
the class StreamsPartitionAssignorTest method shouldGenerateTasksForAllCreatedPartitions.
@Test
public void shouldGenerateTasksForAllCreatedPartitions() throws Exception {
final StreamsBuilder builder = new StreamsBuilder();
final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder);
internalTopologyBuilder.setApplicationId(applicationId);
// KStream with 3 partitions
KStream<Object, Object> stream1 = builder.stream("topic1").map(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {
@Override
public KeyValue<Object, Object> apply(final Object key, final Object value) {
return new KeyValue<>(key, value);
}
});
// KTable with 4 partitions
KTable<Object, Long> table1 = builder.table("topic3").groupBy(new KeyValueMapper<Object, Object, KeyValue<Object, Object>>() {
@Override
public KeyValue<Object, Object> apply(final Object key, final Object value) {
return new KeyValue<>(key, value);
}
}).count();
// joining the stream and the table
// this triggers the enforceCopartitioning() routine in the StreamsPartitionAssignor,
// forcing the stream.map to get repartitioned to a topic with four partitions.
stream1.join(table1, new ValueJoiner() {
@Override
public Object apply(final Object value1, final Object value2) {
return null;
}
});
final UUID uuid = UUID.randomUUID();
final String client = "client1";
mockTaskManager(Collections.<TaskId>emptySet(), Collections.<TaskId>emptySet(), UUID.randomUUID(), internalTopologyBuilder);
configurePartitionAssignor(Collections.<String, Object>emptyMap());
final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer);
partitionAssignor.setInternalTopicManager(mockInternalTopicManager);
final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
final Set<TaskId> emptyTasks = Collections.emptySet();
subscriptions.put(client, new PartitionAssignor.Subscription(Utils.mkList("topic1", "topic3"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
final Map<String, PartitionAssignor.Assignment> assignment = partitionAssignor.assign(metadata, subscriptions);
final Map<String, Integer> expectedCreatedInternalTopics = new HashMap<>();
expectedCreatedInternalTopics.put(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 4);
expectedCreatedInternalTopics.put(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-changelog", 4);
expectedCreatedInternalTopics.put(applicationId + "-KSTREAM-MAP-0000000001-repartition", 4);
expectedCreatedInternalTopics.put(applicationId + "-topic3-STATE-STORE-0000000002-changelog", 4);
// check if all internal topics were created as expected
assertThat(mockInternalTopicManager.readyTopics, equalTo(expectedCreatedInternalTopics));
final List<TopicPartition> expectedAssignment = Arrays.asList(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2), new TopicPartition("topic3", 0), new TopicPartition("topic3", 1), new TopicPartition("topic3", 2), new TopicPartition("topic3", 3), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 0), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 1), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 2), new TopicPartition(applicationId + "-KTABLE-AGGREGATE-STATE-STORE-0000000006-repartition", 3), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 0), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 1), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 2), new TopicPartition(applicationId + "-KSTREAM-MAP-0000000001-repartition", 3));
// check if we created a task for all expected topicPartitions.
assertThat(new HashSet<>(assignment.get(client).partitions()), equalTo(new HashSet<>(expectedAssignment)));
}
use of org.apache.kafka.streams.StreamsBuilder in project apache-kafka-on-k8s by banzaicloud.
the class SimpleBenchmark method createKafkaStreamsKTableKTableJoin.
private KafkaStreams createKafkaStreamsKTableKTableJoin(Properties streamConfig, String kTableTopic1, String kTableTopic2, final CountDownLatch latch) {
final StreamsBuilder builder = new StreamsBuilder();
final KTable<Long, byte[]> input1 = builder.table(kTableTopic1);
final KTable<Long, byte[]> input2 = builder.table(kTableTopic2);
input1.leftJoin(input2, VALUE_JOINER).foreach(new CountDownAction(latch));
return createKafkaStreamsWithExceptionHandler(builder, streamConfig);
}
Aggregations