Examples with TopologyTestDriver org.apache.kafka.streams.TopologyTestDriver used on opensource projects

Example 16 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KGroupedStreamImplTest method doCountSlidingWindows.

private void doCountSlidingWindows(final MockApiProcessorSupplier<Windowed<String>, Long, Void, Void> supplier) {
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
        inputTopic.pipeInput("1", "A", 500L);
        inputTopic.pipeInput("1", "A", 999L);
        inputTopic.pipeInput("1", "A", 600L);
        inputTopic.pipeInput("2", "B", 500L);
        inputTopic.pipeInput("2", "B", 600L);
        inputTopic.pipeInput("2", "B", 700L);
        inputTopic.pipeInput("3", "C", 501L);
        inputTopic.pipeInput("1", "A", 1000L);
        inputTopic.pipeInput("1", "A", 1000L);
        inputTopic.pipeInput("2", "B", 1000L);
        inputTopic.pipeInput("2", "B", 1000L);
        inputTopic.pipeInput("3", "C", 600L);
    }
    final Comparator<KeyValueTimestamp<Windowed<String>, Long>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, Long> o) -> o.key().key()).thenComparing((KeyValueTimestamp<Windowed<String>, Long> o) -> o.key().window().start());
    final ArrayList<KeyValueTimestamp<Windowed<String>, Long>> actual = supplier.theCapturedProcessor().processed();
    actual.sort(comparator);
    assertThat(actual, equalTo(Arrays.asList(// processing A@500
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(0L, 500L)), 1L, 500L), // processing A@600
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(100L, 600L)), 2L, 600L), // processing A@999
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(499L, 999L)), 2L, 999L), // processing A@600
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(499L, 999L)), 3L, 999L), // processing first A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(500L, 1000L)), 4L, 1000L), // processing second A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(500L, 1000L)), 5L, 1000L), // processing A@999
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(501L, 1001L)), 1L, 999L), // processing A@600
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(501L, 1001L)), 2L, 999L), // processing first A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(501L, 1001L)), 3L, 1000L), // processing second A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(501L, 1001L)), 4L, 1000L), // processing A@600
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(601L, 1101L)), 1L, 999L), // processing first A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(601L, 1101L)), 2L, 1000L), // processing second A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(601L, 1101L)), 3L, 1000L), // processing first A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(1000L, 1500L)), 1L, 1000L), // processing second A@1000
    new KeyValueTimestamp<>(new Windowed<>("1", new TimeWindow(1000L, 1500L)), 2L, 1000L), // processing B@500
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(0L, 500L)), 1L, 500L), // processing B@600
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(100L, 600L)), 2L, 600L), // processing B@700
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(200L, 700L)), 3L, 700L), // processing first B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(500L, 1000L)), 4L, 1000L), // processing second B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(500L, 1000L)), 5L, 1000L), // processing B@600
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(501L, 1001L)), 1L, 600L), // processing B@700
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(501L, 1001L)), 2L, 700L), // processing first B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(501L, 1001L)), 3L, 1000L), // processing second B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(501L, 1001L)), 4L, 1000L), // processing B@700
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(601L, 1101L)), 1L, 700L), // processing first B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(601L, 1101)), 2L, 1000L), // processing second B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(601L, 1101)), 3L, 1000L), // processing first B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(701L, 1201L)), 1L, 1000L), // processing second B@1000
    new KeyValueTimestamp<>(new Windowed<>("2", new TimeWindow(701L, 1201L)), 2L, 1000L), // processing C@501
    new KeyValueTimestamp<>(new Windowed<>("3", new TimeWindow(1L, 501L)), 1L, 501L), // processing C@600
    new KeyValueTimestamp<>(new Windowed<>("3", new TimeWindow(100L, 600L)), 2L, 600L), // processing C@600
    new KeyValueTimestamp<>(new Windowed<>("3", new TimeWindow(502L, 1002L)), 1L, 600L))));
}

Example 17 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KGroupedStreamImplTest method doReduceSessionWindows.

private void doReduceSessionWindows(final MockApiProcessorSupplier<Windowed<String>, String, Void, Void> supplier) {
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(TOPIC, new StringSerializer(), new StringSerializer());
        inputTopic.pipeInput("1", "A", 10);
        inputTopic.pipeInput("2", "Z", 15);
        inputTopic.pipeInput("1", "B", 30);
        inputTopic.pipeInput("1", "A", 70);
        inputTopic.pipeInput("1", "B", 100);
        inputTopic.pipeInput("1", "C", 90);
    }
    final Map<Windowed<String>, ValueAndTimestamp<String>> result = supplier.theCapturedProcessor().lastValueAndTimestampPerKey();
    assertEquals(ValueAndTimestamp.make("A:B", 30L), result.get(new Windowed<>("1", new SessionWindow(10L, 30L))));
    assertEquals(ValueAndTimestamp.make("Z", 15L), result.get(new Windowed<>("2", new SessionWindow(15L, 15L))));
    assertEquals(ValueAndTimestamp.make("A:B:C", 100L), result.get(new Windowed<>("1", new SessionWindow(70L, 100L))));
}

Example 18 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KGroupedStreamImplTest method shouldAggregateWithDefaultSerdes.

@Test
public void shouldAggregateWithDefaultSerdes() {
    final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    groupedStream.aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER).toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        processData(driver);
        assertThat(supplier.theCapturedProcessor().lastValueAndTimestampPerKey().get("1"), equalTo(ValueAndTimestamp.make("0+A+C+D", 10L)));
        assertThat(supplier.theCapturedProcessor().lastValueAndTimestampPerKey().get("2"), equalTo(ValueAndTimestamp.make("0+B", 1L)));
        assertThat(supplier.theCapturedProcessor().lastValueAndTimestampPerKey().get("3"), equalTo(ValueAndTimestamp.make("0+E+F", 9L)));
    }
}

Example 19 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KGroupedStreamImplTest method shouldLogAndMeasureSkipsInAggregate.

@Test
public void shouldLogAndMeasureSkipsInAggregate() {
    groupedStream.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("count").withKeySerde(Serdes.String()));
    try (final LogCaptureAppender appender = LogCaptureAppender.createAndRegister(KStreamAggregate.class);
        final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        processData(driver);
        assertThat(appender.getMessages(), hasItem("Skipping record due to null key or value. topic=[topic] partition=[0] " + "offset=[6]"));
    }
}

Example 20 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KStreamBranchTest method testKStreamBranch.

// Old PAPI. Needs to be migrated.
@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testKStreamBranch() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Predicate<Integer, String> isEven = (key, value) -> (key % 2) == 0;
    final Predicate<Integer, String> isMultipleOfThree = (key, value) -> (key % 3) == 0;
    final Predicate<Integer, String> isOdd = (key, value) -> (key % 2) != 0;
    final int[] expectedKeys = new int[] { 1, 2, 3, 4, 5, 6 };
    final KStream<Integer, String> stream;
    final KStream<Integer, String>[] branches;
    stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
    branches = stream.branch(isEven, isMultipleOfThree, isOdd);
    assertEquals(3, branches.length);
    final MockProcessorSupplier<Integer, String> supplier = new MockProcessorSupplier<>();
    for (final KStream<Integer, String> branch : branches) {
        branch.process(supplier);
    }
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, String> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
        for (final int expectedKey : expectedKeys) {
            inputTopic.pipeInput(expectedKey, "V" + expectedKey);
        }
    }
    final List<MockProcessor<Integer, String>> processors = supplier.capturedProcessors(3);
    assertEquals(3, processors.get(0).processed().size());
    assertEquals(1, processors.get(1).processed().size());
    assertEquals(2, processors.get(2).processed().size());
}