Examples with KeyValueMapper org.apache.kafka.streams.kstream.KeyValueMapper used on opensource projects

Example 1 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class ResetIntegrationTest method setupTopologyWithIntermediateUserTopic.

private KStreamBuilder setupTopologyWithIntermediateUserTopic(final String outputTopic2) {
    final KStreamBuilder builder = new KStreamBuilder();
    final KStream<Long, String> input = builder.stream(INPUT_TOPIC);
    // use map to trigger internal re-partitioning before groupByKey
    input.map(new KeyValueMapper<Long, String, KeyValue<Long, String>>() {

        @Override
        public KeyValue<Long, String> apply(final Long key, final String value) {
            return new KeyValue<>(key, value);
        }
    }).groupByKey().count("global-count").to(Serdes.Long(), Serdes.Long(), OUTPUT_TOPIC);
    input.through(INTERMEDIATE_USER_TOPIC).groupByKey().count(TimeWindows.of(35).advanceBy(10), "count").toStream().map(new KeyValueMapper<Windowed<Long>, Long, KeyValue<Long, Long>>() {

        @Override
        public KeyValue<Long, Long> apply(final Windowed<Long> key, final Long value) {
            return new KeyValue<>(key.window().start() + key.window().end(), value);
        }
    }).to(Serdes.Long(), Serdes.Long(), outputTopic2);
    return builder;
}

Example 2 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KGroupedTableImplTest method shouldReduce.

@Test
public void shouldReduce() throws Exception {
    final String topic = "input";
    final KeyValueMapper<String, Number, KeyValue<String, Integer>> intProjection = new KeyValueMapper<String, Number, KeyValue<String, Integer>>() {

        @Override
        public KeyValue<String, Integer> apply(String key, Number value) {
            return KeyValue.pair(key, value.intValue());
        }
    };
    final KTable<String, Integer> reduced = builder.table(Serdes.String(), Serdes.Double(), topic, "store").groupBy(intProjection).reduce(MockReducer.INTEGER_ADDER, MockReducer.INTEGER_SUBTRACTOR, "reduced");
    final Map<String, Integer> results = new HashMap<>();
    reduced.foreach(new ForeachAction<String, Integer>() {

        @Override
        public void apply(final String key, final Integer value) {
            results.put(key, value);
        }
    });
    driver = new KStreamTestDriver(builder, TestUtils.tempDirectory(), Serdes.String(), Serdes.Integer());
    driver.setTime(10L);
    driver.process(topic, "A", 1.1);
    driver.process(topic, "B", 2.2);
    driver.flushState();
    assertEquals(Integer.valueOf(1), results.get("A"));
    assertEquals(Integer.valueOf(2), results.get("B"));
    driver.process(topic, "A", 2.6);
    driver.process(topic, "B", 1.3);
    driver.process(topic, "A", 5.7);
    driver.process(topic, "B", 6.2);
    driver.flushState();
    assertEquals(Integer.valueOf(5), results.get("A"));
    assertEquals(Integer.valueOf(6), results.get("B"));
}

Example 3 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamAggregationDedupIntegrationTest method shouldGroupByKey.

@Test
public void shouldGroupByKey() throws Exception {
    final long timestamp = mockTime.milliseconds();
    produceMessages(timestamp);
    produceMessages(timestamp);
    stream.groupByKey(Serdes.Integer(), Serdes.String()).count(TimeWindows.of(500L), "count-windows").toStream(new KeyValueMapper<Windowed<Integer>, Long, String>() {

        @Override
        public String apply(final Windowed<Integer> windowedKey, final Long value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.Long(), outputTopic);
    startStreams();
    final List<KeyValue<String, Long>> results = receiveMessages(new StringDeserializer(), new LongDeserializer(), 5);
    Collections.sort(results, new Comparator<KeyValue<String, Long>>() {

        @Override
        public int compare(final KeyValue<String, Long> o1, final KeyValue<String, Long> o2) {
            return KStreamAggregationDedupIntegrationTest.compare(o1, o2);
        }
    });
    final long window = timestamp / 500 * 500;
    assertThat(results, is(Arrays.asList(KeyValue.pair("1@" + window, 2L), KeyValue.pair("2@" + window, 2L), KeyValue.pair("3@" + window, 2L), KeyValue.pair("4@" + window, 2L), KeyValue.pair("5@" + window, 2L))));
}

Example 4 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamFlatMapTest method testFlatMap.

@Test
public void testFlatMap() {
    KStreamBuilder builder = new KStreamBuilder();
    KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>> mapper = new KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>>() {

        @Override
        public Iterable<KeyValue<String, String>> apply(Number key, Object value) {
            ArrayList<KeyValue<String, String>> result = new ArrayList<>();
            for (int i = 0; i < key.intValue(); i++) {
                result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString()));
            }
            return result;
        }
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    KStream<Integer, String> stream;
    MockProcessorSupplier<String, String> processor;
    processor = new MockProcessorSupplier<>();
    stream = builder.stream(Serdes.Integer(), Serdes.String(), topicName);
    stream.flatMap(mapper).process(processor);
    driver = new KStreamTestDriver(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(6, processor.processed.size());
    String[] expected = { "10:V1", "20:V2", "21:V2", "30:V3", "31:V3", "32:V3" };
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.processed.get(i));
    }
}

Example 5 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KTableKTableLeftJoinTest method shouldNotThrowIllegalStateExceptionWhenMultiCacheEvictions.

/**
     * This test was written to reproduce https://issues.apache.org/jira/browse/KAFKA-4492
     * It is based on a fairly complicated join used by the developer that reported the bug.
     * Before the fix this would trigger an IllegalStateException.
     */
@Test
public void shouldNotThrowIllegalStateExceptionWhenMultiCacheEvictions() throws Exception {
    final String agg = "agg";
    final String tableOne = "tableOne";
    final String tableTwo = "tableTwo";
    final String tableThree = "tableThree";
    final String tableFour = "tableFour";
    final String tableFive = "tableFive";
    final String tableSix = "tableSix";
    final String[] inputs = { agg, tableOne, tableTwo, tableThree, tableFour, tableFive, tableSix };
    final KStreamBuilder builder = new KStreamBuilder();
    final KTable<Long, String> aggTable = builder.table(Serdes.Long(), Serdes.String(), agg, agg).groupBy(new KeyValueMapper<Long, String, KeyValue<Long, String>>() {

        @Override
        public KeyValue<Long, String> apply(final Long key, final String value) {
            return new KeyValue<>(key, value);
        }
    }, Serdes.Long(), Serdes.String()).reduce(MockReducer.STRING_ADDER, MockReducer.STRING_ADDER, "agg-store");
    final KTable<Long, String> one = builder.table(Serdes.Long(), Serdes.String(), tableOne, tableOne);
    final KTable<Long, String> two = builder.table(Serdes.Long(), Serdes.String(), tableTwo, tableTwo);
    final KTable<Long, String> three = builder.table(Serdes.Long(), Serdes.String(), tableThree, tableThree);
    final KTable<Long, String> four = builder.table(Serdes.Long(), Serdes.String(), tableFour, tableFour);
    final KTable<Long, String> five = builder.table(Serdes.Long(), Serdes.String(), tableFive, tableFive);
    final KTable<Long, String> six = builder.table(Serdes.Long(), Serdes.String(), tableSix, tableSix);
    final ValueMapper<String, String> mapper = new ValueMapper<String, String>() {

        @Override
        public String apply(final String value) {
            return value.toUpperCase(Locale.ROOT);
        }
    };
    final KTable<Long, String> seven = one.mapValues(mapper);
    final KTable<Long, String> eight = six.leftJoin(seven, MockValueJoiner.TOSTRING_JOINER);
    aggTable.leftJoin(one, MockValueJoiner.TOSTRING_JOINER).leftJoin(two, MockValueJoiner.TOSTRING_JOINER).leftJoin(three, MockValueJoiner.TOSTRING_JOINER).leftJoin(four, MockValueJoiner.TOSTRING_JOINER).leftJoin(five, MockValueJoiner.TOSTRING_JOINER).leftJoin(eight, MockValueJoiner.TOSTRING_JOINER).mapValues(mapper);
    driver = new KStreamTestDriver(builder, stateDir, 250);
    final String[] values = { "a", "AA", "BBB", "CCCC", "DD", "EEEEEEEE", "F", "GGGGGGGGGGGGGGG", "HHH", "IIIIIIIIII", "J", "KK", "LLLL", "MMMMMMMMMMMMMMMMMMMMMM", "NNNNN", "O", "P", "QQQQQ", "R", "SSSS", "T", "UU", "VVVVVVVVVVVVVVVVVVV" };
    final Random random = new Random();
    for (int i = 0; i < 1000; i++) {
        for (String input : inputs) {
            final Long key = (long) random.nextInt(1000);
            final String value = values[random.nextInt(values.length)];
            driver.process(input, key, value);
        }
    }
}