Examples with KStreamTestDriver org.apache.kafka.test.KStreamTestDriver used on opensource projects

Example 6 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KStreamBranchTest method testKStreamBranch.

@SuppressWarnings("unchecked")
@Test
public void testKStreamBranch() {
    KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId("X");
    Predicate<Integer, String> isEven = new Predicate<Integer, String>() {

        @Override
        public boolean test(Integer key, String value) {
            return (key % 2) == 0;
        }
    };
    Predicate<Integer, String> isMultipleOfThree = new Predicate<Integer, String>() {

        @Override
        public boolean test(Integer key, String value) {
            return (key % 3) == 0;
        }
    };
    Predicate<Integer, String> isOdd = new Predicate<Integer, String>() {

        @Override
        public boolean test(Integer key, String value) {
            return (key % 2) != 0;
        }
    };
    final int[] expectedKeys = new int[] { 1, 2, 3, 4, 5, 6 };
    KStream<Integer, String> stream;
    KStream<Integer, String>[] branches;
    MockProcessorSupplier<Integer, String>[] processors;
    stream = builder.stream(Serdes.Integer(), Serdes.String(), topicName);
    branches = stream.branch(isEven, isMultipleOfThree, isOdd);
    assertEquals(3, branches.length);
    processors = (MockProcessorSupplier<Integer, String>[]) Array.newInstance(MockProcessorSupplier.class, branches.length);
    for (int i = 0; i < branches.length; i++) {
        processors[i] = new MockProcessorSupplier<>();
        branches[i].process(processors[i]);
    }
    driver = new KStreamTestDriver(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(3, processors[0].processed.size());
    assertEquals(1, processors[1].processed.size());
    assertEquals(2, processors[2].processed.size());
}

Example 7 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KStreamFilterTest method testFilterNot.

@Test
public void testFilterNot() {
    KStreamBuilder builder = new KStreamBuilder();
    final int[] expectedKeys = new int[] { 1, 2, 3, 4, 5, 6, 7 };
    KStream<Integer, String> stream;
    MockProcessorSupplier<Integer, String> processor;
    processor = new MockProcessorSupplier<>();
    stream = builder.stream(Serdes.Integer(), Serdes.String(), topicName);
    stream.filterNot(isMultipleOfThree).process(processor);
    driver = new KStreamTestDriver(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(5, processor.processed.size());
}

Example 8 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KStreamFlatMapTest method testFlatMap.

@Test
public void testFlatMap() {
    KStreamBuilder builder = new KStreamBuilder();
    KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>> mapper = new KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>>() {

        @Override
        public Iterable<KeyValue<String, String>> apply(Number key, Object value) {
            ArrayList<KeyValue<String, String>> result = new ArrayList<>();
            for (int i = 0; i < key.intValue(); i++) {
                result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString()));
            }
            return result;
        }
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    KStream<Integer, String> stream;
    MockProcessorSupplier<String, String> processor;
    processor = new MockProcessorSupplier<>();
    stream = builder.stream(Serdes.Integer(), Serdes.String(), topicName);
    stream.flatMap(mapper).process(processor);
    driver = new KStreamTestDriver(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(6, processor.processed.size());
    String[] expected = { "10:V1", "20:V2", "21:V2", "30:V3", "31:V3", "32:V3" };
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.processed.get(i));
    }
}

Example 9 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KStreamKStreamJoinTest method testAsymetricWindowingBefore.

@Test
public void testAsymetricWindowingBefore() throws Exception {
    long time = 1000L;
    KStreamBuilder builder = new KStreamBuilder();
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    KStream<Integer, String> stream1;
    KStream<Integer, String> stream2;
    KStream<Integer, String> joined;
    MockProcessorSupplier<Integer, String> processor;
    processor = new MockProcessorSupplier<>();
    stream1 = builder.stream(intSerde, stringSerde, topic1);
    stream2 = builder.stream(intSerde, stringSerde, topic2);
    joined = stream1.join(stream2, MockValueJoiner.TOSTRING_JOINER, JoinWindows.of(0).before(100), intSerde, stringSerde, stringSerde);
    joined.process(processor);
    Collection<Set<String>> copartitionGroups = builder.copartitionGroups();
    assertEquals(1, copartitionGroups.size());
    assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
    driver = new KStreamTestDriver(builder, stateDir);
    for (int i = 0; i < expectedKeys.length; i++) {
        setRecordContext(time + i, topic1);
        driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
    }
    processor.checkAndClearProcessResult();
    time = 1000L - 100L - 1L;
    setRecordContext(time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult();
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
    time = 1000L;
    setRecordContext(time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult();
}

Example 10 with KStreamTestDriver

use of org.apache.kafka.test.KStreamTestDriver in project kafka by apache.

the class KStreamKStreamJoinTest method testWindowing.

@Test
public void testWindowing() throws Exception {
    long time = 0L;
    KStreamBuilder builder = new KStreamBuilder();
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    KStream<Integer, String> stream1;
    KStream<Integer, String> stream2;
    KStream<Integer, String> joined;
    MockProcessorSupplier<Integer, String> processor;
    processor = new MockProcessorSupplier<>();
    stream1 = builder.stream(intSerde, stringSerde, topic1);
    stream2 = builder.stream(intSerde, stringSerde, topic2);
    joined = stream1.join(stream2, MockValueJoiner.TOSTRING_JOINER, JoinWindows.of(100), intSerde, stringSerde, stringSerde);
    joined.process(processor);
    Collection<Set<String>> copartitionGroups = builder.copartitionGroups();
    assertEquals(1, copartitionGroups.size());
    assertEquals(new HashSet<>(Arrays.asList(topic1, topic2)), copartitionGroups.iterator().next());
    driver = new KStreamTestDriver(builder, stateDir);
    // push two items to the primary stream. the other window is empty. this should produce no items.
    // w1 = {}
    // w2 = {}
    // --> w1 = { 0:X0, 1:X1 }
    //     w2 = {}
    setRecordContext(time, topic1);
    for (int i = 0; i < 2; i++) {
        driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
    }
    processor.checkAndClearProcessResult();
    // push two items to the other stream. this should produce two items.
    // w1 = { 0:X0, 1:X1 }
    // w2 = {}
    // --> w1 = { 0:X0, 1:X1 }
    //     w2 = { 0:Y0, 1:Y1 }
    setRecordContext(time, topic2);
    for (int i = 0; i < 2; i++) {
        driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
    }
    processor.checkAndClearProcessResult("0:X0+Y0", "1:X1+Y1");
    // clear logically
    time = 1000L;
    setRecordContext(time, topic1);
    for (int i = 0; i < expectedKeys.length; i++) {
        setRecordContext(time + i, topic1);
        driver.process(topic1, expectedKeys[i], "X" + expectedKeys[i]);
    }
    processor.checkAndClearProcessResult();
    // gradually expires items in w1
    // w1 = { 0:X0, 1:X1, 2:X2, 3:X3 }
    time = 1000 + 100L;
    setRecordContext(time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("2:X2+YY2", "3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("3:X3+YY3");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult();
    // go back to the time before expiration
    time = 1000L - 100L - 1L;
    setRecordContext(time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult();
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2");
    setRecordContext(++time, topic2);
    for (int expectedKey : expectedKeys) {
        driver.process(topic2, expectedKey, "YY" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:X0+YY0", "1:X1+YY1", "2:X2+YY2", "3:X3+YY3");
    // clear (logically)
    time = 2000L;
    for (int i = 0; i < expectedKeys.length; i++) {
        setRecordContext(time + i, topic2);
        driver.process(topic2, expectedKeys[i], "Y" + expectedKeys[i]);
    }
    processor.checkAndClearProcessResult();
    // gradually expires items in w2
    // w2 = { 0:Y0, 1:Y1, 2:Y2, 3:Y3 }
    time = 2000L + 100L;
    setRecordContext(time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("2:XX2+Y2", "3:XX3+Y3");
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("3:XX3+Y3");
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult();
    // go back to the time before expiration
    time = 2000L - 100L - 1L;
    setRecordContext(time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult();
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:XX0+Y0");
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1");
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2");
    setRecordContext(++time, topic1);
    for (int expectedKey : expectedKeys) {
        driver.process(topic1, expectedKey, "XX" + expectedKey);
    }
    processor.checkAndClearProcessResult("0:XX0+Y0", "1:XX1+Y1", "2:XX2+Y2", "3:XX3+Y3");
}