Examples with KStreamTestDriver org.apache.kafka.test.KStreamTestDriver used on opensource projects

Example 56 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KStreamWindowAggregateTest method testJoin.

@Test
public void testJoin() throws Exception {
    final File baseDir = Files.createTempDirectory("test").toFile();
    try {
        final KStreamBuilder builder = new KStreamBuilder();
        String topic1 = "topic1";
        String topic2 = "topic2";
        KStream<String, String> stream1 = builder.stream(strSerde, strSerde, topic1);
        KTable<Windowed<String>, String> table1 = stream1.groupByKey(strSerde, strSerde).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic1-Canonized");
        MockProcessorSupplier<Windowed<String>, String> proc1 = new MockProcessorSupplier<>();
        table1.toStream().process(proc1);
        KStream<String, String> stream2 = builder.stream(strSerde, strSerde, topic2);
        KTable<Windowed<String>, String> table2 = stream2.groupByKey(strSerde, strSerde).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, TimeWindows.of(10).advanceBy(5), strSerde, "topic2-Canonized");
        MockProcessorSupplier<Windowed<String>, String> proc2 = new MockProcessorSupplier<>();
        table2.toStream().process(proc2);
        MockProcessorSupplier<Windowed<String>, String> proc3 = new MockProcessorSupplier<>();
        table1.join(table2, new ValueJoiner<String, String, String>() {

            @Override
            public String apply(String p1, String p2) {
                return p1 + "%" + p2;
            }
        }).toStream().process(proc3);
        driver = new KStreamTestDriver(builder, baseDir);
        setRecordContext(0, topic1);
        driver.process(topic1, "A", "1");
        driver.flushState();
        setRecordContext(1, topic1);
        driver.process(topic1, "B", "2");
        driver.flushState();
        setRecordContext(2, topic1);
        driver.process(topic1, "C", "3");
        driver.flushState();
        setRecordContext(3, topic1);
        driver.process(topic1, "D", "4");
        driver.flushState();
        setRecordContext(4, topic1);
        driver.process(topic1, "A", "1");
        driver.flushState();
        proc1.checkAndClearProcessResult("[A@0]:0+1", "[B@0]:0+2", "[C@0]:0+3", "[D@0]:0+4", "[A@0]:0+1+1");
        proc2.checkAndClearProcessResult();
        proc3.checkAndClearProcessResult();
        setRecordContext(5, topic1);
        driver.process(topic1, "A", "1");
        driver.flushState();
        setRecordContext(6, topic1);
        driver.process(topic1, "B", "2");
        driver.flushState();
        setRecordContext(7, topic1);
        driver.process(topic1, "D", "4");
        driver.flushState();
        setRecordContext(8, topic1);
        driver.process(topic1, "B", "2");
        driver.flushState();
        setRecordContext(9, topic1);
        driver.process(topic1, "C", "3");
        driver.flushState();
        proc1.checkAndClearProcessResult("[A@0]:0+1+1+1", "[A@5]:0+1", "[B@0]:0+2+2", "[B@5]:0+2", "[D@0]:0+4+4", "[D@5]:0+4", "[B@0]:0+2+2+2", "[B@5]:0+2+2", "[C@0]:0+3+3", "[C@5]:0+3");
        proc2.checkAndClearProcessResult();
        proc3.checkAndClearProcessResult();
        setRecordContext(0, topic1);
        driver.process(topic2, "A", "a");
        driver.flushState();
        setRecordContext(1, topic1);
        driver.process(topic2, "B", "b");
        driver.flushState();
        setRecordContext(2, topic1);
        driver.process(topic2, "C", "c");
        driver.flushState();
        setRecordContext(3, topic1);
        driver.process(topic2, "D", "d");
        driver.flushState();
        setRecordContext(4, topic1);
        driver.process(topic2, "A", "a");
        driver.flushState();
        proc1.checkAndClearProcessResult();
        proc2.checkAndClearProcessResult("[A@0]:0+a", "[B@0]:0+b", "[C@0]:0+c", "[D@0]:0+d", "[A@0]:0+a+a");
        proc3.checkAndClearProcessResult("[A@0]:0+1+1+1%0+a", "[B@0]:0+2+2+2%0+b", "[C@0]:0+3+3%0+c", "[D@0]:0+4+4%0+d", "[A@0]:0+1+1+1%0+a+a");
        setRecordContext(5, topic1);
        driver.process(topic2, "A", "a");
        driver.flushState();
        setRecordContext(6, topic1);
        driver.process(topic2, "B", "b");
        driver.flushState();
        setRecordContext(7, topic1);
        driver.process(topic2, "D", "d");
        driver.flushState();
        setRecordContext(8, topic1);
        driver.process(topic2, "B", "b");
        driver.flushState();
        setRecordContext(9, topic1);
        driver.process(topic2, "C", "c");
        driver.flushState();
        proc1.checkAndClearProcessResult();
        proc2.checkAndClearProcessResult("[A@0]:0+a+a+a", "[A@5]:0+a", "[B@0]:0+b+b", "[B@5]:0+b", "[D@0]:0+d+d", "[D@5]:0+d", "[B@0]:0+b+b+b", "[B@5]:0+b+b", "[C@0]:0+c+c", "[C@5]:0+c");
        proc3.checkAndClearProcessResult("[A@0]:0+1+1+1%0+a+a+a", "[A@5]:0+1%0+a", "[B@0]:0+2+2+2%0+b+b", "[B@5]:0+2+2%0+b", "[D@0]:0+4+4%0+d+d", "[D@5]:0+4%0+d", "[B@0]:0+2+2+2%0+b+b+b", "[B@5]:0+2+2%0+b+b", "[C@0]:0+3+3%0+c+c", "[C@5]:0+3%0+c");
    } finally {
        Utils.delete(baseDir);
    }
}

Example 57 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KTableAggregateTest method testAggCoalesced.

@Test
public void testAggCoalesced() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    final String topic1 = "topic1";
    final MockProcessorSupplier<String, String> proc = new MockProcessorSupplier<>();
    KTable<String, String> table1 = builder.table(stringSerde, stringSerde, topic1, "anyStoreName");
    KTable<String, String> table2 = table1.groupBy(MockKeyValueMapper.<String, String>NoOpKeyValueMapper(), stringSerde, stringSerde).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, stringSerde, "topic1-Canonized");
    table2.toStream().process(proc);
    driver = new KStreamTestDriver(builder, stateDir);
    driver.process(topic1, "A", "1");
    driver.process(topic1, "A", "3");
    driver.process(topic1, "A", "4");
    driver.flushState();
    assertEquals(Utils.mkList("A:0+4"), proc.processed);
}

Example 58 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KTableAggregateTest method testRemoveOldBeforeAddNew.

@Test
public void testRemoveOldBeforeAddNew() throws IOException {
    final KStreamBuilder builder = new KStreamBuilder();
    final String input = "count-test-input";
    final MockProcessorSupplier<String, String> proc = new MockProcessorSupplier<>();
    builder.table(Serdes.String(), Serdes.String(), input, "anyStoreName").groupBy(new KeyValueMapper<String, String, KeyValue<String, String>>() {

        @Override
        public KeyValue<String, String> apply(String key, String value) {
            return KeyValue.pair(String.valueOf(key.charAt(0)), String.valueOf(key.charAt(1)));
        }
    }, stringSerde, stringSerde).aggregate(new Initializer<String>() {

        @Override
        public String apply() {
            return "";
        }
    }, new Aggregator<String, String, String>() {

        @Override
        public String apply(String aggKey, String value, String aggregate) {
            return aggregate + value;
        }
    }, new Aggregator<String, String, String>() {

        @Override
        public String apply(String key, String value, String aggregate) {
            return aggregate.replaceAll(value, "");
        }
    }, Serdes.String(), "someStore").toStream().process(proc);
    driver = new KStreamTestDriver(builder, stateDir);
    driver.process(input, "11", "A");
    driver.flushState();
    driver.process(input, "12", "B");
    driver.flushState();
    driver.process(input, "11", null);
    driver.flushState();
    driver.process(input, "12", "C");
    driver.flushState();
    assertEquals(Utils.mkList("1:1", "1:12", "1:2", "1:2"), proc.processed);
}

Example 59 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KTableFilterTest method testSendingOldValue.

@Test
public void testSendingOldValue() throws IOException {
    KStreamBuilder builder = new KStreamBuilder();
    String topic1 = "topic1";
    KTableImpl<String, Integer, Integer> table1 = (KTableImpl<String, Integer, Integer>) builder.table(stringSerde, intSerde, topic1, "anyStoreName");
    KTableImpl<String, Integer, Integer> table2 = (KTableImpl<String, Integer, Integer>) table1.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    table2.enableSendingOldValues();
    MockProcessorSupplier<String, Integer> proc1 = new MockProcessorSupplier<>();
    MockProcessorSupplier<String, Integer> proc2 = new MockProcessorSupplier<>();
    builder.addProcessor("proc1", proc1, table1.name);
    builder.addProcessor("proc2", proc2, table2.name);
    driver = new KStreamTestDriver(builder, stateDir, null, null);
    driver.process(topic1, "A", 1);
    driver.process(topic1, "B", 1);
    driver.process(topic1, "C", 1);
    driver.flushState();
    proc1.checkAndClearProcessResult("A:(1<-null)", "B:(1<-null)", "C:(1<-null)");
    proc2.checkEmptyAndClearProcessResult();
    driver.process(topic1, "A", 2);
    driver.process(topic1, "B", 2);
    driver.flushState();
    proc1.checkAndClearProcessResult("A:(2<-1)", "B:(2<-1)");
    proc2.checkAndClearProcessResult("A:(2<-null)", "B:(2<-null)");
    driver.process(topic1, "A", 3);
    driver.flushState();
    proc1.checkAndClearProcessResult("A:(3<-2)");
    proc2.checkAndClearProcessResult("A:(null<-2)");
    driver.process(topic1, "A", null);
    driver.process(topic1, "B", null);
    driver.flushState();
    proc1.checkAndClearProcessResult("A:(null<-3)", "B:(null<-2)");
    proc2.checkAndClearProcessResult("B:(null<-2)");
}

Example 60 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KTableFilterTest method testValueGetter.

@Test
public void testValueGetter() throws IOException {
    KStreamBuilder builder = new KStreamBuilder();
    String topic1 = "topic1";
    KTableImpl<String, Integer, Integer> table1 = (KTableImpl<String, Integer, Integer>) builder.table(stringSerde, intSerde, topic1, "anyStoreName");
    KTableImpl<String, Integer, Integer> table2 = (KTableImpl<String, Integer, Integer>) table1.filter(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    KTableImpl<String, Integer, Integer> table3 = (KTableImpl<String, Integer, Integer>) table1.filterNot(new Predicate<String, Integer>() {

        @Override
        public boolean test(String key, Integer value) {
            return (value % 2) == 0;
        }
    });
    KTableValueGetterSupplier<String, Integer> getterSupplier2 = table2.valueGetterSupplier();
    KTableValueGetterSupplier<String, Integer> getterSupplier3 = table3.valueGetterSupplier();
    driver = new KStreamTestDriver(builder, stateDir, null, null);
    KTableValueGetter<String, Integer> getter2 = getterSupplier2.get();
    KTableValueGetter<String, Integer> getter3 = getterSupplier3.get();
    getter2.init(driver.context());
    getter3.init(driver.context());
    driver.process(topic1, "A", 1);
    driver.process(topic1, "B", 1);
    driver.process(topic1, "C", 1);
    assertNull(getter2.get("A"));
    assertNull(getter2.get("B"));
    assertNull(getter2.get("C"));
    assertEquals(1, (int) getter3.get("A"));
    assertEquals(1, (int) getter3.get("B"));
    assertEquals(1, (int) getter3.get("C"));
    driver.process(topic1, "A", 2);
    driver.process(topic1, "B", 2);
    assertEquals(2, (int) getter2.get("A"));
    assertEquals(2, (int) getter2.get("B"));
    assertNull(getter2.get("C"));
    assertNull(getter3.get("A"));
    assertNull(getter3.get("B"));
    assertEquals(1, (int) getter3.get("C"));
    driver.process(topic1, "A", 3);
    assertNull(getter2.get("A"));
    assertEquals(2, (int) getter2.get("B"));
    assertNull(getter2.get("C"));
    assertEquals(3, (int) getter3.get("A"));
    assertNull(getter3.get("B"));
    assertEquals(1, (int) getter3.get("C"));
    driver.process(topic1, "A", null);
    driver.process(topic1, "B", null);
    assertNull(getter2.get("A"));
    assertNull(getter2.get("B"));
    assertNull(getter2.get("C"));
    assertNull(getter3.get("A"));
    assertNull(getter3.get("B"));
    assertEquals(1, (int) getter3.get("C"));
}