Examples with TopologyTestDriver org.apache.kafka.streams.TopologyTestDriver used on opensource projects

Example 31 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KStreamImplTest method shouldProcessFromSourcesThatMatchMultiplePattern.

@Test
public void shouldProcessFromSourcesThatMatchMultiplePattern() {
    final String topic3 = "topic-without-pattern";
    final KStream<String, String> pattern2Source1 = builder.stream(Pattern.compile("topic-\\d"));
    final KStream<String, String> pattern2Source2 = builder.stream(Pattern.compile("topic-[A-Z]"));
    final KStream<String, String> source3 = builder.stream(topic3);
    final KStream<String, String> merged = pattern2Source1.merge(pattern2Source2).merge(source3);
    merged.process(processorSupplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic3 = driver.createInputTopic("topic-3", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopic4 = driver.createInputTopic("topic-4", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopicA = driver.createInputTopic("topic-A", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopicZ = driver.createInputTopic("topic-Z", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(topic3, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        inputTopic3.pipeInput("A", "aa", 1L);
        inputTopic4.pipeInput("B", "bb", 5L);
        inputTopicA.pipeInput("C", "cc", 10L);
        inputTopicZ.pipeInput("D", "dd", 8L);
        inputTopic.pipeInput("E", "ee", 3L);
    }
    assertEquals(asList(new KeyValueTimestamp<>("A", "aa", 1), new KeyValueTimestamp<>("B", "bb", 5), new KeyValueTimestamp<>("C", "cc", 10), new KeyValueTimestamp<>("D", "dd", 8), new KeyValueTimestamp<>("E", "ee", 3)), processorSupplier.theCapturedProcessor().processed());
}

Example 32 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KStreamImplTest method shouldNotMaterializedKTableFromKStream.

@Test
public void shouldNotMaterializedKTableFromKStream() {
    final Consumed<String, String> consumed = Consumed.with(Serdes.String(), Serdes.String());
    final StreamsBuilder builder = new StreamsBuilder();
    final String input = "input";
    final String output = "output";
    builder.stream(input, consumed).toTable().toStream().to(output);
    final String topologyDescription = builder.build().describe().toString();
    assertThat(topologyDescription, equalTo("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [input])\n" + "      --> KSTREAM-TOTABLE-0000000001\n" + "    Processor: KSTREAM-TOTABLE-0000000001 (stores: [])\n" + "      --> KTABLE-TOSTREAM-0000000003\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Processor: KTABLE-TOSTREAM-0000000003 (stores: [])\n" + "      --> KSTREAM-SINK-0000000004\n" + "      <-- KSTREAM-TOTABLE-0000000001\n" + "    Sink: KSTREAM-SINK-0000000004 (topic: output)\n" + "      <-- KTABLE-TOSTREAM-0000000003\n\n"));
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(input, Serdes.String().serializer(), Serdes.String().serializer());
        final TestOutputTopic<String, String> outputTopic = driver.createOutputTopic(output, Serdes.String().deserializer(), Serdes.String().deserializer());
        inputTopic.pipeInput("A", "01", 5L);
        inputTopic.pipeInput("B", "02", 100L);
        inputTopic.pipeInput("C", "03", 0L);
        inputTopic.pipeInput("D", "04", 0L);
        inputTopic.pipeInput("A", "05", 10L);
        inputTopic.pipeInput("A", "06", 8L);
        final List<TestRecord<String, String>> outputExpectRecords = new ArrayList<>();
        outputExpectRecords.add(new TestRecord<>("A", "01", Instant.ofEpochMilli(5L)));
        outputExpectRecords.add(new TestRecord<>("B", "02", Instant.ofEpochMilli(100L)));
        outputExpectRecords.add(new TestRecord<>("C", "03", Instant.ofEpochMilli(0L)));
        outputExpectRecords.add(new TestRecord<>("D", "04", Instant.ofEpochMilli(0L)));
        outputExpectRecords.add(new TestRecord<>("A", "05", Instant.ofEpochMilli(10L)));
        outputExpectRecords.add(new TestRecord<>("A", "06", Instant.ofEpochMilli(8L)));
        assertEquals(outputTopic.readRecordsToList(), outputExpectRecords);
    }
}

Example 33 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KStreamFlatMapTest method testFlatMap.

@Test
public void testFlatMap() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topicName = "topic";
    final KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>> mapper = (key, value) -> {
        final ArrayList<KeyValue<String, String>> result = new ArrayList<>();
        for (int i = 0; i < key.intValue(); i++) {
            result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString()));
        }
        return result;
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    final KStream<Integer, String> stream;
    final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
    stream.flatMap(mapper).process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, String> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0), Duration.ZERO);
        for (final int expectedKey : expectedKeys) {
            inputTopic.pipeInput(expectedKey, "V" + expectedKey);
        }
    }
    assertEquals(6, supplier.theCapturedProcessor().processed().size());
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>("10", "V1", 0), new KeyValueTimestamp<>("20", "V2", 0), new KeyValueTimestamp<>("21", "V2", 0), new KeyValueTimestamp<>("30", "V3", 0), new KeyValueTimestamp<>("31", "V3", 0), new KeyValueTimestamp<>("32", "V3", 0) };
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], supplier.theCapturedProcessor().processed().get(i));
    }
}

Example 34 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KStreamGlobalKTableLeftJoinTest method setUp.

@Before
public void setUp() {
    builder = new StreamsBuilder();
    final KStream<Integer, String> stream;
    // value of stream optionally contains key of table
    final GlobalKTable<String, String> table;
    final KeyValueMapper<Integer, String, String> keyMapper;
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    final Consumed<Integer, String> streamConsumed = Consumed.with(Serdes.Integer(), Serdes.String());
    final Consumed<String, String> tableConsumed = Consumed.with(Serdes.String(), Serdes.String());
    stream = builder.stream(streamTopic, streamConsumed);
    table = builder.globalTable(globalTableTopic, tableConsumed);
    keyMapper = (key, value) -> {
        final String[] tokens = value.split(",");
        // If not present, use null to indicate no match
        return tokens.length > 1 ? tokens[1] : null;
    };
    stream.leftJoin(table, keyMapper, MockValueJoiner.TOSTRING_JOINER).process(supplier);
    final Properties props = StreamsTestUtils.getStreamsConfig(Serdes.Integer(), Serdes.String());
    driver = new TopologyTestDriver(builder.build(), props);
    processor = supplier.theCapturedProcessor();
}

Example 35 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KStreamImplValueJoinerWithKeyTest method runJoinTopology.

private void runJoinTopology(final StreamsBuilder builder, final List<KeyValue<String, String>> expectedResults, final boolean isTableJoin, final String rightTopic) {
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, Integer> rightInputTopic = driver.createInputTopic(rightTopic, new StringSerializer(), new IntegerSerializer());
        final TestInputTopic<String, Integer> leftInputTopic = driver.createInputTopic(leftTopic, new StringSerializer(), new IntegerSerializer());
        final TestOutputTopic<String, String> joinResultTopic = driver.createOutputTopic(outputTopic, new StringDeserializer(), new StringDeserializer());
        // the table first, joins only triggered from streams side
        if (isTableJoin) {
            rightInputTopic.pipeInput("A", 2);
            leftInputTopic.pipeInput("A", 3);
        } else {
            leftInputTopic.pipeInput("A", 3);
            rightInputTopic.pipeInput("A", 2);
        }
        final List<KeyValue<String, String>> actualResult = joinResultTopic.readKeyValuesToList();
        assertEquals(expectedResults, actualResult);
    }
}