Examples with MockApiProcessor org.apache.kafka.test.MockApiProcessor used on opensource projects

Example 6 with MockApiProcessor

use of org.apache.kafka.test.MockApiProcessor in project kafka by apache.

the class KStreamSlidingWindowAggregateTest method testJoin.

@Test
public void testJoin() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String topic2 = "topic2";
    final WindowBytesStoreSupplier storeSupplier1 = inOrderIterator ? new InOrderMemoryWindowStoreSupplier("InOrder1", 50000L, 10L, false) : Stores.inMemoryWindowStore("Reverse1", Duration.ofMillis(50000), Duration.ofMillis(10), false);
    final WindowBytesStoreSupplier storeSupplier2 = inOrderIterator ? new InOrderMemoryWindowStoreSupplier("InOrder2", 50000L, 10L, false) : Stores.inMemoryWindowStore("Reverse2", Duration.ofMillis(50000), Duration.ofMillis(10), false);
    final KTable<Windowed<String>, String> table1 = builder.stream(topic1, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(SlidingWindows.ofTimeDifferenceAndGrace(ofMillis(10), ofMillis(100))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.as(storeSupplier1));
    final KTable<Windowed<String>, String> table2 = builder.stream(topic2, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(SlidingWindows.ofTimeDifferenceAndGrace(ofMillis(10), ofMillis(100))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.as(storeSupplier2));
    final MockApiProcessorSupplier<Windowed<String>, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1.toStream().process(supplier);
    table2.toStream().process(supplier);
    table1.join(table2, (p1, p2) -> p1 + "%" + p2).toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> inputTopic2 = driver.createInputTopic(topic2, new StringSerializer(), new StringSerializer());
        inputTopic1.pipeInput("A", "1", 10L);
        inputTopic1.pipeInput("B", "2", 11L);
        inputTopic1.pipeInput("C", "3", 12L);
        final List<MockApiProcessor<Windowed<String>, String, Void, Void>> processors = supplier.capturedProcessors(3);
        processors.get(0).checkAndClearProcessResult(// left windows created by the first set of records to table 1
        new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1", 10), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(1, 11)), "0+2", 11), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(2, 12)), "0+3", 12));
        processors.get(1).checkAndClearProcessResult();
        processors.get(2).checkAndClearProcessResult();
        inputTopic1.pipeInput("A", "1", 15L);
        inputTopic1.pipeInput("B", "2", 16L);
        inputTopic1.pipeInput("C", "3", 19L);
        processors.get(0).checkAndClearProcessResult(// right windows from previous records are created, and left windows from new records to table 1
        new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(11, 21)), "0+1", 15), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+1", 15), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(12, 22)), "0+2", 16), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(6, 16)), "0+2+2", 16), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(13, 23)), "0+3", 19), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(9, 19)), "0+3+3", 19));
        processors.get(1).checkAndClearProcessResult();
        processors.get(2).checkAndClearProcessResult();
        inputTopic2.pipeInput("A", "a", 10L);
        inputTopic2.pipeInput("B", "b", 30L);
        inputTopic2.pipeInput("C", "c", 12L);
        inputTopic2.pipeInput("C", "c", 35L);
        processors.get(0).checkAndClearProcessResult();
        processors.get(1).checkAndClearProcessResult(// left windows from first set of records sent to table 2
        new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+a", 10), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(20, 30)), "0+b", 30), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(2, 12)), "0+c", 12), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(25, 35)), "0+c", 35));
        processors.get(2).checkAndClearProcessResult(// set of join windows from windows created by table 1 and table 2
        new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1%0+a", 10), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(2, 12)), "0+3%0+c", 12));
        inputTopic2.pipeInput("A", "a", 15L);
        inputTopic2.pipeInput("B", "b", 16L);
        inputTopic2.pipeInput("C", "c", 17L);
        processors.get(0).checkAndClearProcessResult();
        processors.get(1).checkAndClearProcessResult(// right windows from previous records are created (where applicable), and left windows from new records to table 2
        new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(11, 21)), "0+a", 15), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+a+a", 15), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(6, 16)), "0+b", 16), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(13, 23)), "0+c", 17), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(7, 17)), "0+c+c", 17));
        processors.get(2).checkAndClearProcessResult(// set of join windows from windows created by table 1 and table 2
        new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(11, 21)), "0+1%0+a", 15), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+1%0+a+a", 15), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(6, 16)), "0+2+2%0+b", 16), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(13, 23)), "0+3%0+c", 19));
    }
}

Example 7 with MockApiProcessor

use of org.apache.kafka.test.MockApiProcessor in project kafka by apache.

the class KStreamWindowAggregateTest method testJoin.

@Test
public void testJoin() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String topic2 = "topic2";
    final KTable<Windowed<String>, String> table1 = builder.stream(topic1, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(TimeWindows.ofSizeAndGrace(ofMillis(10), ofMillis(100)).advanceBy(ofMillis(5))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.<String, String, WindowStore<Bytes, byte[]>>as("topic1-Canonized").withValueSerde(Serdes.String()));
    final MockApiProcessorSupplier<Windowed<String>, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1.toStream().process(supplier);
    final KTable<Windowed<String>, String> table2 = builder.stream(topic2, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(TimeWindows.ofSizeAndGrace(ofMillis(10), ofMillis(100)).advanceBy(ofMillis(5))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.<String, String, WindowStore<Bytes, byte[]>>as("topic2-Canonized").withValueSerde(Serdes.String()));
    table2.toStream().process(supplier);
    table1.join(table2, (p1, p2) -> p1 + "%" + p2).toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> inputTopic2 = driver.createInputTopic(topic2, new StringSerializer(), new StringSerializer());
        inputTopic1.pipeInput("A", "1", 0L);
        inputTopic1.pipeInput("B", "2", 1L);
        inputTopic1.pipeInput("C", "3", 2L);
        inputTopic1.pipeInput("D", "4", 3L);
        inputTopic1.pipeInput("A", "1", 9L);
        final List<MockApiProcessor<Windowed<String>, String, Void, Void>> processors = supplier.capturedProcessors(3);
        processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1", 0), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2", 1), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3", 2), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4", 3), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1", 9), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1", 9));
        processors.get(1).checkAndClearProcessResult();
        processors.get(2).checkAndClearProcessResult();
        inputTopic1.pipeInput("A", "1", 5L);
        inputTopic1.pipeInput("B", "2", 6L);
        inputTopic1.pipeInput("D", "4", 7L);
        inputTopic1.pipeInput("B", "2", 8L);
        inputTopic1.pipeInput("C", "3", 9L);
        processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1+1", 9), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+1", 9), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2", 6), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2", 6), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4+4", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+4", 7), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2", 8), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2", 8), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3+3", 9), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(5, 15)), "0+3", 9));
        processors.get(1).checkAndClearProcessResult();
        processors.get(2).checkAndClearProcessResult();
        inputTopic2.pipeInput("A", "a", 0L);
        inputTopic2.pipeInput("B", "b", 1L);
        inputTopic2.pipeInput("C", "c", 2L);
        inputTopic2.pipeInput("D", "d", 20L);
        inputTopic2.pipeInput("A", "a", 20L);
        processors.get(0).checkAndClearProcessResult();
        processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+a", 0), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+b", 1), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+c", 2), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(15, 25)), "0+d", 20), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(20, 30)), "0+d", 20), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(15, 25)), "0+a", 20), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(20, 30)), "0+a", 20));
        processors.get(2).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1+1%0+a", 9), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2%0+b", 8), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3+3%0+c", 9));
        inputTopic2.pipeInput("A", "a", 5L);
        inputTopic2.pipeInput("B", "b", 6L);
        inputTopic2.pipeInput("D", "d", 7L);
        inputTopic2.pipeInput("D", "d", 18L);
        inputTopic2.pipeInput("A", "a", 21L);
        processors.get(0).checkAndClearProcessResult();
        processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+a+a", 5), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+a", 5), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+b+b", 6), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+b", 6), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+d", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+d", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(10, 20)), "0+d", 18), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(15, 25)), "0+d+d", 20), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(15, 25)), "0+a+a", 21), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(20, 30)), "0+a+a", 21));
        processors.get(2).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1+1%0+a+a", 9), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+1%0+a", 9), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2%0+b+b", 8), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2%0+b", 8), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4+4%0+d", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+4%0+d", 7));
    }
}

Example 8 with MockApiProcessor

use of org.apache.kafka.test.MockApiProcessor in project kafka by apache.

the class KTableAggregateTest method testRemoveOldBeforeAddNew.

@Test
public void testRemoveOldBeforeAddNew() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String input = "count-test-input";
    final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    builder.table(input, consumed).groupBy((key, value) -> KeyValue.pair(String.valueOf(key.charAt(0)), String.valueOf(key.charAt(1))), stringSerialized).aggregate(() -> "", (aggKey, value, aggregate) -> aggregate + value, (key, value, aggregate) -> aggregate.replaceAll(value, ""), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("someStore").withValueSerde(Serdes.String())).toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), CONFIG, Instant.ofEpochMilli(0L))) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final MockApiProcessor<String, String, Void, Void> proc = supplier.theCapturedProcessor();
        inputTopic.pipeInput("11", "A", 10L);
        inputTopic.pipeInput("12", "B", 8L);
        inputTopic.pipeInput("11", (String) null, 12L);
        inputTopic.pipeInput("12", "C", 6L);
        assertEquals(asList(new KeyValueTimestamp<>("1", "1", 10), new KeyValueTimestamp<>("1", "12", 10), new KeyValueTimestamp<>("1", "2", 12), new KeyValueTimestamp<>("1", "", 12), new KeyValueTimestamp<>("1", "2", 12L)), proc.processed());
    }
}

Example 9 with MockApiProcessor

use of org.apache.kafka.test.MockApiProcessor in project kafka by apache.

the class KTableImplTest method testKTable.

@Test
public void testKTable() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String topic2 = "topic2";
    final KTable<String, String> table1 = builder.table(topic1, consumed);
    final MockApiProcessorSupplier<String, Object, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1.toStream().process(supplier);
    final KTable<String, Integer> table2 = table1.mapValues(s -> Integer.valueOf(s));
    table2.toStream().process(supplier);
    final KTable<String, Integer> table3 = table2.filter((key, value) -> (value % 2) == 0);
    table3.toStream().process(supplier);
    table1.toStream().to(topic2, produced);
    final KTable<String, String> table4 = builder.table(topic2, consumed);
    table4.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        inputTopic.pipeInput("A", "01", 5L);
        inputTopic.pipeInput("B", "02", 100L);
        inputTopic.pipeInput("C", "03", 0L);
        inputTopic.pipeInput("D", "04", 0L);
        inputTopic.pipeInput("A", "05", 10L);
        inputTopic.pipeInput("A", "06", 8L);
    }
    final List<MockApiProcessor<String, Object, Void, Void>> processors = supplier.capturedProcessors(4);
    assertEquals(asList(new KeyValueTimestamp<>("A", "01", 5), new KeyValueTimestamp<>("B", "02", 100), new KeyValueTimestamp<>("C", "03", 0), new KeyValueTimestamp<>("D", "04", 0), new KeyValueTimestamp<>("A", "05", 10), new KeyValueTimestamp<>("A", "06", 8)), processors.get(0).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", 1, 5), new KeyValueTimestamp<>("B", 2, 100), new KeyValueTimestamp<>("C", 3, 0), new KeyValueTimestamp<>("D", 4, 0), new KeyValueTimestamp<>("A", 5, 10), new KeyValueTimestamp<>("A", 6, 8)), processors.get(1).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", null, 5), new KeyValueTimestamp<>("B", 2, 100), new KeyValueTimestamp<>("C", null, 0), new KeyValueTimestamp<>("D", 4, 0), new KeyValueTimestamp<>("A", null, 10), new KeyValueTimestamp<>("A", 6, 8)), processors.get(2).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", "01", 5), new KeyValueTimestamp<>("B", "02", 100), new KeyValueTimestamp<>("C", "03", 0), new KeyValueTimestamp<>("D", "04", 0), new KeyValueTimestamp<>("A", "05", 10), new KeyValueTimestamp<>("A", "06", 8)), processors.get(3).processed());
}

Example 10 with MockApiProcessor

use of org.apache.kafka.test.MockApiProcessor in project kafka by apache.

the class KTableFilterTest method doTestKTable.

private void doTestKTable(final StreamsBuilder builder, final KTable<String, Integer> table2, final KTable<String, Integer> table3, final String topic) {
    final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table2.toStream().process(supplier);
    table3.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, Integer> inputTopic = driver.createInputTopic(topic, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        inputTopic.pipeInput("A", 1, 10L);
        inputTopic.pipeInput("B", 2, 5L);
        inputTopic.pipeInput("C", 3, 8L);
        inputTopic.pipeInput("D", 4, 14L);
        inputTopic.pipeInput("A", null, 18L);
        inputTopic.pipeInput("B", null, 15L);
    }
    final List<MockApiProcessor<String, Integer, Void, Void>> processors = supplier.capturedProcessors(2);
    processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", null, 10), new KeyValueTimestamp<>("B", 2, 5), new KeyValueTimestamp<>("C", null, 8), new KeyValueTimestamp<>("D", 4, 14), new KeyValueTimestamp<>("A", null, 18), new KeyValueTimestamp<>("B", null, 15));
    processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>("A", 1, 10), new KeyValueTimestamp<>("B", null, 5), new KeyValueTimestamp<>("C", 3, 8), new KeyValueTimestamp<>("D", null, 14), new KeyValueTimestamp<>("A", null, 18), new KeyValueTimestamp<>("B", null, 15));
}