Examples with KeyValueMapper org.apache.kafka.streams.kstream.KeyValueMapper used on opensource projects

Example 11 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldGroupByKey.

@Test
public void shouldGroupByKey() throws Exception {
    final long timestamp = mockTime.milliseconds();
    produceMessages(timestamp);
    produceMessages(timestamp);
    stream.groupByKey(Serdes.Integer(), Serdes.String()).count(TimeWindows.of(500L), "count-windows").toStream(new KeyValueMapper<Windowed<Integer>, Long, String>() {

        @Override
        public String apply(final Windowed<Integer> windowedKey, final Long value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.Long(), outputTopic);
    startStreams();
    final List<KeyValue<String, Long>> results = receiveMessages(new StringDeserializer(), new LongDeserializer(), 10);
    Collections.sort(results, new Comparator<KeyValue<String, Long>>() {

        @Override
        public int compare(final KeyValue<String, Long> o1, final KeyValue<String, Long> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    });
    final long window = timestamp / 500 * 500;
    assertThat(results, is(Arrays.asList(KeyValue.pair("1@" + window, 1L), KeyValue.pair("1@" + window, 2L), KeyValue.pair("2@" + window, 1L), KeyValue.pair("2@" + window, 2L), KeyValue.pair("3@" + window, 1L), KeyValue.pair("3@" + window, 2L), KeyValue.pair("4@" + window, 1L), KeyValue.pair("4@" + window, 2L), KeyValue.pair("5@" + window, 1L), KeyValue.pair("5@" + window, 2L))));
}

Example 12 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldReduceWindowed.

@Test
public void shouldReduceWindowed() throws Exception {
    final long firstBatchTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = mockTime.milliseconds();
    produceMessages(secondBatchTimestamp);
    produceMessages(secondBatchTimestamp);
    groupedStream.reduce(reducer, TimeWindows.of(500L), "reduce-time-windows").toStream(new KeyValueMapper<Windowed<String>, String, String>() {

        @Override
        public String apply(final Windowed<String> windowedKey, final String value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.String(), outputTopic);
    startStreams();
    final List<KeyValue<String, String>> windowedOutput = receiveMessages(new StringDeserializer(), new StringDeserializer(), 15);
    final Comparator<KeyValue<String, String>> comparator = new Comparator<KeyValue<String, String>>() {

        @Override
        public int compare(final KeyValue<String, String> o1, final KeyValue<String, String> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    };
    Collections.sort(windowedOutput, comparator);
    final long firstBatchWindow = firstBatchTimestamp / 500 * 500;
    final long secondBatchWindow = secondBatchTimestamp / 500 * 500;
    assertThat(windowedOutput, is(Arrays.asList(new KeyValue<>("A@" + firstBatchWindow, "A"), new KeyValue<>("A@" + secondBatchWindow, "A"), new KeyValue<>("A@" + secondBatchWindow, "A:A"), new KeyValue<>("B@" + firstBatchWindow, "B"), new KeyValue<>("B@" + secondBatchWindow, "B"), new KeyValue<>("B@" + secondBatchWindow, "B:B"), new KeyValue<>("C@" + firstBatchWindow, "C"), new KeyValue<>("C@" + secondBatchWindow, "C"), new KeyValue<>("C@" + secondBatchWindow, "C:C"), new KeyValue<>("D@" + firstBatchWindow, "D"), new KeyValue<>("D@" + secondBatchWindow, "D"), new KeyValue<>("D@" + secondBatchWindow, "D:D"), new KeyValue<>("E@" + firstBatchWindow, "E"), new KeyValue<>("E@" + secondBatchWindow, "E"), new KeyValue<>("E@" + secondBatchWindow, "E:E"))));
}

Example 13 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldAggregateWindowed.

@Test
public void shouldAggregateWindowed() throws Exception {
    final long firstTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstTimestamp);
    final long secondTimestamp = mockTime.milliseconds();
    produceMessages(secondTimestamp);
    produceMessages(secondTimestamp);
    groupedStream.aggregate(initializer, aggregator, TimeWindows.of(500L), Serdes.Integer(), "aggregate-by-key-windowed").toStream(new KeyValueMapper<Windowed<String>, Integer, String>() {

        @Override
        public String apply(final Windowed<String> windowedKey, final Integer value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(Serdes.String(), Serdes.Integer(), outputTopic);
    startStreams();
    final List<KeyValue<String, Integer>> windowedMessages = receiveMessages(new StringDeserializer(), new IntegerDeserializer(), 15);
    final Comparator<KeyValue<String, Integer>> comparator = new Comparator<KeyValue<String, Integer>>() {

        @Override
        public int compare(final KeyValue<String, Integer> o1, final KeyValue<String, Integer> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    };
    Collections.sort(windowedMessages, comparator);
    final long firstWindow = firstTimestamp / 500 * 500;
    final long secondWindow = secondTimestamp / 500 * 500;
    assertThat(windowedMessages, is(Arrays.asList(new KeyValue<>("A@" + firstWindow, 1), new KeyValue<>("A@" + secondWindow, 1), new KeyValue<>("A@" + secondWindow, 2), new KeyValue<>("B@" + firstWindow, 1), new KeyValue<>("B@" + secondWindow, 1), new KeyValue<>("B@" + secondWindow, 2), new KeyValue<>("C@" + firstWindow, 1), new KeyValue<>("C@" + secondWindow, 1), new KeyValue<>("C@" + secondWindow, 2), new KeyValue<>("D@" + firstWindow, 1), new KeyValue<>("D@" + secondWindow, 1), new KeyValue<>("D@" + secondWindow, 2), new KeyValue<>("E@" + firstWindow, 1), new KeyValue<>("E@" + secondWindow, 1), new KeyValue<>("E@" + secondWindow, 2))));
}

Example 14 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamMapTest method testTypeVariance.

@Test
public void testTypeVariance() throws Exception {
    KeyValueMapper<Number, Object, KeyValue<Number, String>> stringify = new KeyValueMapper<Number, Object, KeyValue<Number, String>>() {

        @Override
        public KeyValue<Number, String> apply(Number key, Object value) {
            return KeyValue.pair(key, key + ":" + value);
        }
    };
    new KStreamBuilder().<Integer, String>stream("numbers").map(stringify).to("strings");
}

Example 15 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KStreamSelectKeyTest method testSelectKey.

@Test
public void testSelectKey() {
    KStreamBuilder builder = new KStreamBuilder();
    final Map<Number, String> keyMap = new HashMap<>();
    keyMap.put(1, "ONE");
    keyMap.put(2, "TWO");
    keyMap.put(3, "THREE");
    KeyValueMapper<Object, Number, String> selector = new KeyValueMapper<Object, Number, String>() {

        @Override
        public String apply(Object key, Number value) {
            return keyMap.get(value);
        }
    };
    final String[] expected = new String[] { "ONE:1", "TWO:2", "THREE:3" };
    final int[] expectedValues = new int[] { 1, 2, 3 };
    KStream<String, Integer> stream = builder.stream(stringSerde, integerSerde, topicName);
    MockProcessorSupplier<String, Integer> processor = new MockProcessorSupplier<>();
    stream.selectKey(selector).process(processor);
    driver = new KStreamTestDriver(builder);
    for (int expectedValue : expectedValues) {
        driver.process(topicName, null, expectedValue);
    }
    assertEquals(3, processor.processed.size());
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.processed.get(i));
    }
}