use of org.apache.kafka.streams.KeyValue in project kafka by apache.
the class CogroupedKStreamImplTest method shouldInsertRepartitionsTopicForUpstreamKeyModificationWithGroupedReusedInSameCogroupsWithOptimization.
@Test
public void shouldInsertRepartitionsTopicForUpstreamKeyModificationWithGroupedReusedInSameCogroupsWithOptimization() {
final Properties properties = new Properties();
properties.setProperty(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.OPTIMIZE);
final StreamsBuilder builder = new StreamsBuilder();
final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
final KStream<String, String> stream2 = builder.stream("two", stringConsumed);
final KGroupedStream<String, String> groupedOne = stream1.map((k, v) -> new KeyValue<>(v, k)).groupByKey();
final KGroupedStream<String, String> groupedTwo = stream2.groupByKey();
final KTable<String, String> cogroupedTwo = groupedOne.cogroup(STRING_AGGREGATOR).cogroup(groupedTwo, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER);
final KTable<String, String> cogroupedOne = groupedOne.cogroup(STRING_AGGREGATOR).cogroup(groupedTwo, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER);
cogroupedOne.toStream().to(OUTPUT);
cogroupedTwo.toStream().to("OUTPUT2");
final String topologyDescription = builder.build(properties).describe().toString();
assertThat(topologyDescription, equalTo("Topologies:\n" + " Sub-topology: 0\n" + " Source: KSTREAM-SOURCE-0000000000 (topics: [one])\n" + " --> KSTREAM-MAP-0000000002\n" + " Processor: KSTREAM-MAP-0000000002 (stores: [])\n" + " --> COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter\n" + " <-- KSTREAM-SOURCE-0000000000\n" + " Processor: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter (stores: [])\n" + " --> COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-sink\n" + " <-- KSTREAM-MAP-0000000002\n" + " Sink: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-sink (topic: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition)\n" + " <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter\n\n" + " Sub-topology: 1\n" + " Source: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source (topics: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition])\n" + " --> COGROUPKSTREAM-AGGREGATE-0000000014, COGROUPKSTREAM-AGGREGATE-0000000007\n" + " Source: KSTREAM-SOURCE-0000000001 (topics: [two])\n" + " --> COGROUPKSTREAM-AGGREGATE-0000000015, COGROUPKSTREAM-AGGREGATE-0000000008\n" + " Processor: COGROUPKSTREAM-AGGREGATE-0000000007 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003])\n" + " --> COGROUPKSTREAM-MERGE-0000000009\n" + " <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source\n" + " Processor: COGROUPKSTREAM-AGGREGATE-0000000008 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003])\n" + " --> COGROUPKSTREAM-MERGE-0000000009\n" + " <-- KSTREAM-SOURCE-0000000001\n" + " Processor: COGROUPKSTREAM-AGGREGATE-0000000014 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000010])\n" + " --> COGROUPKSTREAM-MERGE-0000000016\n" + " <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source\n" + " Processor: COGROUPKSTREAM-AGGREGATE-0000000015 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000010])\n" + " --> COGROUPKSTREAM-MERGE-0000000016\n" + " <-- KSTREAM-SOURCE-0000000001\n" + " Processor: COGROUPKSTREAM-MERGE-0000000009 (stores: [])\n" + " --> KTABLE-TOSTREAM-0000000019\n" + " <-- COGROUPKSTREAM-AGGREGATE-0000000007, COGROUPKSTREAM-AGGREGATE-0000000008\n" + " Processor: COGROUPKSTREAM-MERGE-0000000016 (stores: [])\n" + " --> KTABLE-TOSTREAM-0000000017\n" + " <-- COGROUPKSTREAM-AGGREGATE-0000000014, COGROUPKSTREAM-AGGREGATE-0000000015\n" + " Processor: KTABLE-TOSTREAM-0000000017 (stores: [])\n" + " --> KSTREAM-SINK-0000000018\n" + " <-- COGROUPKSTREAM-MERGE-0000000016\n" + " Processor: KTABLE-TOSTREAM-0000000019 (stores: [])\n" + " --> KSTREAM-SINK-0000000020\n" + " <-- COGROUPKSTREAM-MERGE-0000000009\n" + " Sink: KSTREAM-SINK-0000000018 (topic: output)\n" + " <-- KTABLE-TOSTREAM-0000000017\n" + " Sink: KSTREAM-SINK-0000000020 (topic: OUTPUT2)\n" + " <-- KTABLE-TOSTREAM-0000000019\n\n"));
}
use of org.apache.kafka.streams.KeyValue in project kafka by apache.
the class CogroupedKStreamImplTest method shouldInsertRepartitionsTopicForUpstreamKeyModificationWithGroupedReused.
@Test
public void shouldInsertRepartitionsTopicForUpstreamKeyModificationWithGroupedReused() {
final StreamsBuilder builder = new StreamsBuilder();
final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
final KStream<String, String> stream2 = builder.stream("two", stringConsumed);
final KGroupedStream<String, String> groupedOne = stream1.map((k, v) -> new KeyValue<>(v, k)).groupByKey();
final KGroupedStream<String, String> groupedTwo = stream2.groupByKey();
groupedOne.cogroup(STRING_AGGREGATOR).cogroup(groupedTwo, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER);
groupedOne.aggregate(STRING_INITIALIZER, STRING_AGGREGATOR);
final String topologyDescription = builder.build().describe().toString();
assertThat(topologyDescription, equalTo("Topologies:\n" + " Sub-topology: 0\n" + " Source: KSTREAM-SOURCE-0000000000 (topics: [one])\n" + " --> KSTREAM-MAP-0000000002\n" + " Processor: KSTREAM-MAP-0000000002 (stores: [])\n" + " --> COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter, KSTREAM-FILTER-0000000013\n" + " <-- KSTREAM-SOURCE-0000000000\n" + " Processor: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter (stores: [])\n" + " --> COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-sink\n" + " <-- KSTREAM-MAP-0000000002\n" + " Processor: KSTREAM-FILTER-0000000013 (stores: [])\n" + " --> KSTREAM-SINK-0000000012\n" + " <-- KSTREAM-MAP-0000000002\n" + " Sink: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-sink (topic: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition)\n" + " <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-filter\n" + " Sink: KSTREAM-SINK-0000000012 (topic: KSTREAM-AGGREGATE-STATE-STORE-0000000010-repartition)\n" + " <-- KSTREAM-FILTER-0000000013\n\n" + " Sub-topology: 1\n" + " Source: COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source (topics: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition])\n" + " --> COGROUPKSTREAM-AGGREGATE-0000000007\n" + " Source: KSTREAM-SOURCE-0000000001 (topics: [two])\n" + " --> COGROUPKSTREAM-AGGREGATE-0000000008\n" + " Processor: COGROUPKSTREAM-AGGREGATE-0000000007 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003])\n" + " --> COGROUPKSTREAM-MERGE-0000000009\n" + " <-- COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003-repartition-source\n" + " Processor: COGROUPKSTREAM-AGGREGATE-0000000008 (stores: [COGROUPKSTREAM-AGGREGATE-STATE-STORE-0000000003])\n" + " --> COGROUPKSTREAM-MERGE-0000000009\n" + " <-- KSTREAM-SOURCE-0000000001\n" + " Processor: COGROUPKSTREAM-MERGE-0000000009 (stores: [])\n" + " --> none\n" + " <-- COGROUPKSTREAM-AGGREGATE-0000000007, COGROUPKSTREAM-AGGREGATE-0000000008\n\n" + " Sub-topology: 2\n" + " Source: KSTREAM-SOURCE-0000000014 (topics: [KSTREAM-AGGREGATE-STATE-STORE-0000000010-repartition])\n" + " --> KSTREAM-AGGREGATE-0000000011\n" + " Processor: KSTREAM-AGGREGATE-0000000011 (stores: [KSTREAM-AGGREGATE-STATE-STORE-0000000010])\n" + " --> none\n" + " <-- KSTREAM-SOURCE-0000000014\n\n"));
}
use of org.apache.kafka.streams.KeyValue in project kafka by apache.
the class KStreamKStreamJoinTest method shouldCreateRepartitionTopicsWithUserProvidedName.
@Test
public void shouldCreateRepartitionTopicsWithUserProvidedName() {
final StreamsBuilder builder = new StreamsBuilder();
final Properties props = new Properties();
props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.NO_OPTIMIZATION);
final KStream<String, String> stream1 = builder.stream("topic", Consumed.with(Serdes.String(), Serdes.String()));
final KStream<String, String> stream2 = builder.stream("topic2", Consumed.with(Serdes.String(), Serdes.String()));
final KStream<String, String> stream3 = builder.stream("topic3", Consumed.with(Serdes.String(), Serdes.String()));
final KStream<String, String> newStream = stream1.map((k, v) -> new KeyValue<>(v, k));
final StreamJoined<String, String, String> streamJoined = StreamJoined.with(Serdes.String(), Serdes.String(), Serdes.String());
newStream.join(stream2, (value1, value2) -> value1 + value2, JoinWindows.ofTimeDifferenceWithNoGrace(ofMillis(100)), streamJoined.withName("first-join")).to("out-one");
newStream.join(stream3, (value1, value2) -> value1 + value2, JoinWindows.ofTimeDifferenceWithNoGrace(ofMillis(100)), streamJoined.withName("second-join")).to("out-two");
final Topology topology = builder.build(props);
System.out.println(topology.describe().toString());
assertEquals(expectedTopologyWithUserNamedRepartitionTopics, topology.describe().toString());
}
use of org.apache.kafka.streams.KeyValue in project kafka by apache.
the class KStreamImplTest method shouldSupportKeyChangeKTableFromKStream.
@Test
public void shouldSupportKeyChangeKTableFromKStream() {
final Consumed<String, String> consumed = Consumed.with(Serdes.String(), Serdes.String());
final StreamsBuilder builder = new StreamsBuilder();
final String input = "input";
final String output = "output";
builder.stream(input, consumed).map((key, value) -> new KeyValue<>(key.charAt(0) - 'A', value)).toTable(Materialized.with(Serdes.Integer(), null)).toStream().to(output);
final Topology topology = builder.build();
final String topologyDescription = topology.describe().toString();
assertThat(topologyDescription, equalTo("Topologies:\n" + " Sub-topology: 0\n" + " Source: KSTREAM-SOURCE-0000000000 (topics: [input])\n" + " --> KSTREAM-MAP-0000000001\n" + " Processor: KSTREAM-MAP-0000000001 (stores: [])\n" + " --> KSTREAM-FILTER-0000000005\n" + " <-- KSTREAM-SOURCE-0000000000\n" + " Processor: KSTREAM-FILTER-0000000005 (stores: [])\n" + " --> KSTREAM-SINK-0000000004\n" + " <-- KSTREAM-MAP-0000000001\n" + " Sink: KSTREAM-SINK-0000000004 (topic: KSTREAM-TOTABLE-0000000002-repartition)\n" + " <-- KSTREAM-FILTER-0000000005\n" + "\n" + " Sub-topology: 1\n" + " Source: KSTREAM-SOURCE-0000000006 (topics: [KSTREAM-TOTABLE-0000000002-repartition])\n" + " --> KSTREAM-TOTABLE-0000000002\n" + " Processor: KSTREAM-TOTABLE-0000000002 (stores: [])\n" + " --> KTABLE-TOSTREAM-0000000007\n" + " <-- KSTREAM-SOURCE-0000000006\n" + " Processor: KTABLE-TOSTREAM-0000000007 (stores: [])\n" + " --> KSTREAM-SINK-0000000008\n" + " <-- KSTREAM-TOTABLE-0000000002\n" + " Sink: KSTREAM-SINK-0000000008 (topic: output)\n" + " <-- KTABLE-TOSTREAM-0000000007\n\n"));
try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
final TestInputTopic<String, String> inputTopic = driver.createInputTopic(input, Serdes.String().serializer(), Serdes.String().serializer());
final TestOutputTopic<Integer, String> outputTopic = driver.createOutputTopic(output, Serdes.Integer().deserializer(), Serdes.String().deserializer());
inputTopic.pipeInput("A", "01", 5L);
inputTopic.pipeInput("B", "02", 100L);
inputTopic.pipeInput("C", "03", 0L);
inputTopic.pipeInput("D", "04", 0L);
inputTopic.pipeInput("A", "05", 10L);
inputTopic.pipeInput("A", "06", 8L);
final List<TestRecord<Integer, String>> outputExpectRecords = new ArrayList<>();
outputExpectRecords.add(new TestRecord<>(0, "01", Instant.ofEpochMilli(5L)));
outputExpectRecords.add(new TestRecord<>(1, "02", Instant.ofEpochMilli(100L)));
outputExpectRecords.add(new TestRecord<>(2, "03", Instant.ofEpochMilli(0L)));
outputExpectRecords.add(new TestRecord<>(3, "04", Instant.ofEpochMilli(0L)));
outputExpectRecords.add(new TestRecord<>(0, "05", Instant.ofEpochMilli(10L)));
outputExpectRecords.add(new TestRecord<>(0, "06", Instant.ofEpochMilli(8L)));
assertEquals(outputTopic.readRecordsToList(), outputExpectRecords);
}
}
use of org.apache.kafka.streams.KeyValue in project kafka by apache.
the class KStreamFlatMapTest method testFlatMap.
@Test
public void testFlatMap() {
final StreamsBuilder builder = new StreamsBuilder();
final String topicName = "topic";
final KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>> mapper = (key, value) -> {
final ArrayList<KeyValue<String, String>> result = new ArrayList<>();
for (int i = 0; i < key.intValue(); i++) {
result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString()));
}
return result;
};
final int[] expectedKeys = { 0, 1, 2, 3 };
final KStream<Integer, String> stream;
final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
stream.flatMap(mapper).process(supplier);
try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
final TestInputTopic<Integer, String> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0), Duration.ZERO);
for (final int expectedKey : expectedKeys) {
inputTopic.pipeInput(expectedKey, "V" + expectedKey);
}
}
assertEquals(6, supplier.theCapturedProcessor().processed().size());
final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>("10", "V1", 0), new KeyValueTimestamp<>("20", "V2", 0), new KeyValueTimestamp<>("21", "V2", 0), new KeyValueTimestamp<>("30", "V3", 0), new KeyValueTimestamp<>("31", "V3", 0), new KeyValueTimestamp<>("32", "V3", 0) };
for (int i = 0; i < expected.length; i++) {
assertEquals(expected[i], supplier.theCapturedProcessor().processed().get(i));
}
}
Aggregations