Examples with MockApiProcessorSupplier org.apache.kafka.test.MockApiProcessorSupplier used on opensource projects

Example 51 with MockApiProcessorSupplier

use of org.apache.kafka.test.MockApiProcessorSupplier in project kafka by apache.

the class KStreamWindowAggregateTest method testJoin.

@Test
public void testJoin() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String topic2 = "topic2";
    final KTable<Windowed<String>, String> table1 = builder.stream(topic1, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(TimeWindows.ofSizeAndGrace(ofMillis(10), ofMillis(100)).advanceBy(ofMillis(5))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.<String, String, WindowStore<Bytes, byte[]>>as("topic1-Canonized").withValueSerde(Serdes.String()));
    final MockApiProcessorSupplier<Windowed<String>, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1.toStream().process(supplier);
    final KTable<Windowed<String>, String> table2 = builder.stream(topic2, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(TimeWindows.ofSizeAndGrace(ofMillis(10), ofMillis(100)).advanceBy(ofMillis(5))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.<String, String, WindowStore<Bytes, byte[]>>as("topic2-Canonized").withValueSerde(Serdes.String()));
    table2.toStream().process(supplier);
    table1.join(table2, (p1, p2) -> p1 + "%" + p2).toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> inputTopic2 = driver.createInputTopic(topic2, new StringSerializer(), new StringSerializer());
        inputTopic1.pipeInput("A", "1", 0L);
        inputTopic1.pipeInput("B", "2", 1L);
        inputTopic1.pipeInput("C", "3", 2L);
        inputTopic1.pipeInput("D", "4", 3L);
        inputTopic1.pipeInput("A", "1", 9L);
        final List<MockApiProcessor<Windowed<String>, String, Void, Void>> processors = supplier.capturedProcessors(3);
        processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1", 0), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2", 1), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3", 2), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4", 3), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1", 9), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1", 9));
        processors.get(1).checkAndClearProcessResult();
        processors.get(2).checkAndClearProcessResult();
        inputTopic1.pipeInput("A", "1", 5L);
        inputTopic1.pipeInput("B", "2", 6L);
        inputTopic1.pipeInput("D", "4", 7L);
        inputTopic1.pipeInput("B", "2", 8L);
        inputTopic1.pipeInput("C", "3", 9L);
        processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1+1", 9), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+1", 9), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2", 6), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2", 6), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4+4", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+4", 7), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2", 8), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2", 8), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3+3", 9), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(5, 15)), "0+3", 9));
        processors.get(1).checkAndClearProcessResult();
        processors.get(2).checkAndClearProcessResult();
        inputTopic2.pipeInput("A", "a", 0L);
        inputTopic2.pipeInput("B", "b", 1L);
        inputTopic2.pipeInput("C", "c", 2L);
        inputTopic2.pipeInput("D", "d", 20L);
        inputTopic2.pipeInput("A", "a", 20L);
        processors.get(0).checkAndClearProcessResult();
        processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+a", 0), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+b", 1), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+c", 2), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(15, 25)), "0+d", 20), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(20, 30)), "0+d", 20), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(15, 25)), "0+a", 20), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(20, 30)), "0+a", 20));
        processors.get(2).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1+1%0+a", 9), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2%0+b", 8), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3+3%0+c", 9));
        inputTopic2.pipeInput("A", "a", 5L);
        inputTopic2.pipeInput("B", "b", 6L);
        inputTopic2.pipeInput("D", "d", 7L);
        inputTopic2.pipeInput("D", "d", 18L);
        inputTopic2.pipeInput("A", "a", 21L);
        processors.get(0).checkAndClearProcessResult();
        processors.get(1).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+a+a", 5), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+a", 5), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+b+b", 6), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+b", 6), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+d", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+d", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(10, 20)), "0+d", 18), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(15, 25)), "0+d+d", 20), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(15, 25)), "0+a+a", 21), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(20, 30)), "0+a+a", 21));
        processors.get(2).checkAndClearProcessResult(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1+1%0+a+a", 9), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+1%0+a", 9), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2%0+b+b", 8), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2%0+b", 8), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4+4%0+d", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+4%0+d", 7));
    }
}

Example 52 with MockApiProcessorSupplier

use of org.apache.kafka.test.MockApiProcessorSupplier in project kafka by apache.

the class KTableAggregateTest method testRemoveOldBeforeAddNew.

@Test
public void testRemoveOldBeforeAddNew() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String input = "count-test-input";
    final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    builder.table(input, consumed).groupBy((key, value) -> KeyValue.pair(String.valueOf(key.charAt(0)), String.valueOf(key.charAt(1))), stringSerialized).aggregate(() -> "", (aggKey, value, aggregate) -> aggregate + value, (key, value, aggregate) -> aggregate.replaceAll(value, ""), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("someStore").withValueSerde(Serdes.String())).toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), CONFIG, Instant.ofEpochMilli(0L))) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final MockApiProcessor<String, String, Void, Void> proc = supplier.theCapturedProcessor();
        inputTopic.pipeInput("11", "A", 10L);
        inputTopic.pipeInput("12", "B", 8L);
        inputTopic.pipeInput("11", (String) null, 12L);
        inputTopic.pipeInput("12", "C", 6L);
        assertEquals(asList(new KeyValueTimestamp<>("1", "1", 10), new KeyValueTimestamp<>("1", "12", 10), new KeyValueTimestamp<>("1", "2", 12), new KeyValueTimestamp<>("1", "", 12), new KeyValueTimestamp<>("1", "2", 12L)), proc.processed());
    }
}

Example 53 with MockApiProcessorSupplier

use of org.apache.kafka.test.MockApiProcessorSupplier in project kafka by apache.

the class KTableImplTest method testKTable.

@Test
public void testKTable() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String topic2 = "topic2";
    final KTable<String, String> table1 = builder.table(topic1, consumed);
    final MockApiProcessorSupplier<String, Object, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1.toStream().process(supplier);
    final KTable<String, Integer> table2 = table1.mapValues(s -> Integer.valueOf(s));
    table2.toStream().process(supplier);
    final KTable<String, Integer> table3 = table2.filter((key, value) -> (value % 2) == 0);
    table3.toStream().process(supplier);
    table1.toStream().to(topic2, produced);
    final KTable<String, String> table4 = builder.table(topic2, consumed);
    table4.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        inputTopic.pipeInput("A", "01", 5L);
        inputTopic.pipeInput("B", "02", 100L);
        inputTopic.pipeInput("C", "03", 0L);
        inputTopic.pipeInput("D", "04", 0L);
        inputTopic.pipeInput("A", "05", 10L);
        inputTopic.pipeInput("A", "06", 8L);
    }
    final List<MockApiProcessor<String, Object, Void, Void>> processors = supplier.capturedProcessors(4);
    assertEquals(asList(new KeyValueTimestamp<>("A", "01", 5), new KeyValueTimestamp<>("B", "02", 100), new KeyValueTimestamp<>("C", "03", 0), new KeyValueTimestamp<>("D", "04", 0), new KeyValueTimestamp<>("A", "05", 10), new KeyValueTimestamp<>("A", "06", 8)), processors.get(0).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", 1, 5), new KeyValueTimestamp<>("B", 2, 100), new KeyValueTimestamp<>("C", 3, 0), new KeyValueTimestamp<>("D", 4, 0), new KeyValueTimestamp<>("A", 5, 10), new KeyValueTimestamp<>("A", 6, 8)), processors.get(1).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", null, 5), new KeyValueTimestamp<>("B", 2, 100), new KeyValueTimestamp<>("C", null, 0), new KeyValueTimestamp<>("D", 4, 0), new KeyValueTimestamp<>("A", null, 10), new KeyValueTimestamp<>("A", 6, 8)), processors.get(2).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", "01", 5), new KeyValueTimestamp<>("B", "02", 100), new KeyValueTimestamp<>("C", "03", 0), new KeyValueTimestamp<>("D", "04", 0), new KeyValueTimestamp<>("A", "05", 10), new KeyValueTimestamp<>("A", "06", 8)), processors.get(3).processed());
}

Example 54 with MockApiProcessorSupplier

use of org.apache.kafka.test.MockApiProcessorSupplier in project kafka by apache.

the class KTableKTableInnerJoinTest method testQueryableNotSendingOldValues.

@Test
public void testQueryableNotSendingOldValues() {
    final StreamsBuilder builder = new StreamsBuilder();
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    final KTable<Integer, String> table1;
    final KTable<Integer, String> table2;
    final KTable<Integer, String> joined;
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1 = builder.table(topic1, consumed);
    table2 = builder.table(topic2, consumed);
    joined = table1.join(table2, MockValueJoiner.TOSTRING_JOINER, materialized);
    builder.build().addProcessor("proc", supplier, ((KTableImpl<?, ?, ?>) joined).name);
    doTestNotSendingOldValues(builder, expectedKeys, table1, table2, supplier, joined);
}

Example 55 with MockApiProcessorSupplier

use of org.apache.kafka.test.MockApiProcessorSupplier in project kafka by apache.

the class KStreamKStreamOuterJoinTest method testRightExpiredNonJoinedRecordsAreEmittedByTheRightProcessor.

@Test
public void testRightExpiredNonJoinedRecordsAreEmittedByTheRightProcessor() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, String> stream1;
    final KStream<Integer, String> stream2;
    final KStream<Integer, String> joined;
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    stream1 = builder.stream(topic1, consumed);
    stream2 = builder.stream(topic2, consumed);
    joined = stream1.outerJoin(stream2, MockValueJoiner.TOSTRING_JOINER, JoinWindows.ofTimeDifferenceAndGrace(ofMillis(100L), ofMillis(0L)), StreamJoined.with(Serdes.Integer(), Serdes.String(), Serdes.String()));
    joined.process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, String> inputTopic1 = driver.createInputTopic(topic1, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<Integer, String> inputTopic2 = driver.createInputTopic(topic2, new IntegerSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final MockApiProcessor<Integer, String, Void, Void> processor = supplier.theCapturedProcessor();
        final long windowStart = 0L;
        // No joins detected; No null-joins emitted
        inputTopic2.pipeInput(0, "A0", windowStart + 1L);
        inputTopic2.pipeInput(1, "A1", windowStart + 2L);
        inputTopic2.pipeInput(0, "A0-0", windowStart + 3L);
        processor.checkAndClearProcessResult();
        // Join detected; No null-joins emitted
        inputTopic1.pipeInput(1, "a1", windowStart + 3L);
        processor.checkAndClearProcessResult(new KeyValueTimestamp<>(1, "a1+A1", windowStart + 3L));
        // Dummy record in right topic will emit expired non-joined records from the right topic
        inputTopic2.pipeInput(2, "dummy", windowStart + 401L);
        processor.checkAndClearProcessResult(new KeyValueTimestamp<>(0, "null+A0", windowStart + 1L), new KeyValueTimestamp<>(0, "null+A0-0", windowStart + 3L));
        // Process the dummy joined record
        inputTopic1.pipeInput(2, "dummy", windowStart + 402L);
        processor.checkAndClearProcessResult(new KeyValueTimestamp<>(2, "dummy+dummy", windowStart + 402L));
    }
}