Examples with KeyValueMapper org.apache.kafka.streams.kstream.KeyValueMapper used on opensource projects

Example 6 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamMapTest method testMap.

@Test
public void testMap() {
    KStreamBuilder builder = new KStreamBuilder();
    KeyValueMapper<Integer, String, KeyValue<String, Integer>> mapper = new KeyValueMapper<Integer, String, KeyValue<String, Integer>>() {

        @Override
        public KeyValue<String, Integer> apply(Integer key, String value) {
            return KeyValue.pair(value, key);
        }
    };
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    KStream<Integer, String> stream = builder.stream(intSerde, stringSerde, topicName);
    MockProcessorSupplier<String, Integer> processor;
    processor = new MockProcessorSupplier<>();
    stream.map(mapper).process(processor);
    driver = new KStreamTestDriver(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(4, processor.processed.size());
    String[] expected = new String[] { "V0:0", "V1:1", "V2:2", "V3:3" };
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.processed.get(i));
    }
}

Example 7 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KTableAggregateTest method shouldForwardToCorrectProcessorNodeWhenMultiCacheEvictions.

@Test
public void shouldForwardToCorrectProcessorNodeWhenMultiCacheEvictions() throws Exception {
    final String tableOne = "tableOne";
    final String tableTwo = "tableTwo";
    final KStreamBuilder builder = new KStreamBuilder();
    final String reduceTopic = "TestDriver-reducer-store-repartition";
    final Map<String, Long> reduceResults = new HashMap<>();
    final KTable<String, String> one = builder.table(Serdes.String(), Serdes.String(), tableOne, tableOne);
    final KTable<Long, String> two = builder.table(Serdes.Long(), Serdes.String(), tableTwo, tableTwo);
    final KTable<String, Long> reduce = two.groupBy(new KeyValueMapper<Long, String, KeyValue<String, Long>>() {

        @Override
        public KeyValue<String, Long> apply(final Long key, final String value) {
            return new KeyValue<>(value, key);
        }
    }, Serdes.String(), Serdes.Long()).reduce(new Reducer<Long>() {

        @Override
        public Long apply(final Long value1, final Long value2) {
            return value1 + value2;
        }
    }, new Reducer<Long>() {

        @Override
        public Long apply(final Long value1, final Long value2) {
            return value1 - value2;
        }
    }, "reducer-store");
    reduce.foreach(new ForeachAction<String, Long>() {

        @Override
        public void apply(final String key, final Long value) {
            reduceResults.put(key, value);
        }
    });
    one.leftJoin(reduce, new ValueJoiner<String, Long, String>() {

        @Override
        public String apply(final String value1, final Long value2) {
            return value1 + ":" + value2;
        }
    }).mapValues(new ValueMapper<String, String>() {

        @Override
        public String apply(final String value) {
            return value;
        }
    });
    driver = new KStreamTestDriver(builder, stateDir, 111);
    driver.process(reduceTopic, "1", new Change<>(1L, null));
    driver.process("tableOne", "2", "2");
    // this should trigger eviction on the reducer-store topic
    driver.process(reduceTopic, "2", new Change<>(2L, null));
    // this wont as it is the same value
    driver.process(reduceTopic, "2", new Change<>(2L, null));
    assertEquals(Long.valueOf(2L), reduceResults.get("2"));
    // this will trigger eviction on the tableOne topic
    // that in turn will cause an eviction on reducer-topic. It will flush
    // key 2 as it is the only dirty entry in the cache
    driver.process("tableOne", "1", "5");
    assertEquals(Long.valueOf(4L), reduceResults.get("2"));
}

Example 8 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KTableAggregateTest method testAggRepartition.

@Test
public void testAggRepartition() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    final String topic1 = "topic1";
    final MockProcessorSupplier<String, String> proc = new MockProcessorSupplier<>();
    KTable<String, String> table1 = builder.table(stringSerde, stringSerde, topic1, "anyStoreName");
    KTable<String, String> table2 = table1.groupBy(new KeyValueMapper<String, String, KeyValue<String, String>>() {

        @Override
        public KeyValue<String, String> apply(String key, String value) {
            switch(key) {
                case "null":
                    return KeyValue.pair(null, value);
                case "NULL":
                    return null;
                default:
                    return KeyValue.pair(value, value);
            }
        }
    }, stringSerde, stringSerde).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, stringSerde, "topic1-Canonized");
    table2.toStream().process(proc);
    driver = new KStreamTestDriver(builder, stateDir);
    driver.process(topic1, "A", "1");
    driver.flushState();
    driver.process(topic1, "A", null);
    driver.flushState();
    driver.process(topic1, "A", "1");
    driver.flushState();
    driver.process(topic1, "B", "2");
    driver.flushState();
    driver.process(topic1, "null", "3");
    driver.flushState();
    driver.process(topic1, "B", "4");
    driver.flushState();
    driver.process(topic1, "NULL", "5");
    driver.flushState();
    driver.process(topic1, "B", "7");
    driver.flushState();
    assertEquals(Utils.mkList("1:0+1", "1:0+1-1", "1:0+1-1+1", "2:0+2", //noop
    "2:0+2-2", "4:0+4", //noop
    "4:0+4-4", "7:0+7"), proc.processed);
}

Example 9 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class ResetIntegrationTest method setupTopologyWithoutIntermediateUserTopic.

private KStreamBuilder setupTopologyWithoutIntermediateUserTopic() {
    final KStreamBuilder builder = new KStreamBuilder();
    final KStream<Long, String> input = builder.stream(INPUT_TOPIC);
    // use map to trigger internal re-partitioning before groupByKey
    input.map(new KeyValueMapper<Long, String, KeyValue<Long, Long>>() {

        @Override
        public KeyValue<Long, Long> apply(final Long key, final String value) {
            return new KeyValue<>(key, key);
        }
    }).to(Serdes.Long(), Serdes.Long(), OUTPUT_TOPIC);
    return builder;
}

Example 10 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamAggregationDedupIntegrationTest method shouldReduceWindowed.

@Test
public void shouldReduceWindowed() throws Exception {
    long firstBatchTimestamp = System.currentTimeMillis() - 1000;
    produceMessages(firstBatchTimestamp);
    long secondBatchTimestamp = System.currentTimeMillis();
    produceMessages(secondBatchTimestamp);
    produceMessages(secondBatchTimestamp);
    groupedStream.reduce(reducer, TimeWindows.of(500L), "reduce-time-windows").toStream(new KeyValueMapper<Windowed<String>, String, String>() {

        @Override
        public String apply(Windowed<String> windowedKey, String value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.String(), outputTopic);
    startStreams();
    List<KeyValue<String, String>> windowedOutput = receiveMessages(new StringDeserializer(), new StringDeserializer(), 10);
    Comparator<KeyValue<String, String>> comparator = new Comparator<KeyValue<String, String>>() {

        @Override
        public int compare(final KeyValue<String, String> o1, final KeyValue<String, String> o2) {
            return KStreamAggregationDedupIntegrationTest.compare(o1, o2);
        }
    };
    Collections.sort(windowedOutput, comparator);
    long firstBatchWindow = firstBatchTimestamp / 500 * 500;
    long secondBatchWindow = secondBatchTimestamp / 500 * 500;
    assertThat(windowedOutput, is(Arrays.asList(new KeyValue<>("A@" + firstBatchWindow, "A"), new KeyValue<>("A@" + secondBatchWindow, "A:A"), new KeyValue<>("B@" + firstBatchWindow, "B"), new KeyValue<>("B@" + secondBatchWindow, "B:B"), new KeyValue<>("C@" + firstBatchWindow, "C"), new KeyValue<>("C@" + secondBatchWindow, "C:C"), new KeyValue<>("D@" + firstBatchWindow, "D"), new KeyValue<>("D@" + secondBatchWindow, "D:D"), new KeyValue<>("E@" + firstBatchWindow, "E"), new KeyValue<>("E@" + secondBatchWindow, "E:E"))));
}