Examples with KeyValueMapper org.apache.kafka.streams.kstream.KeyValueMapper used on opensource projects

Example 16 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class WordCountDemo method main.

public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    // setting offset reset to earliest so that we can re-run the demo code with the same pre-loaded data
    // Note: To re-run the demo, you need to use the offset reset tool:
    // https://cwiki.apache.org/confluence/display/KAFKA/Kafka+Streams+Application+Reset+Tool
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> source = builder.stream("streams-file-input");
    KTable<String, Long> counts = source.flatMapValues(new ValueMapper<String, Iterable<String>>() {

        @Override
        public Iterable<String> apply(String value) {
            return Arrays.asList(value.toLowerCase(Locale.getDefault()).split(" "));
        }
    }).map(new KeyValueMapper<String, String, KeyValue<String, String>>() {

        @Override
        public KeyValue<String, String> apply(String key, String value) {
            return new KeyValue<>(value, value);
        }
    }).groupByKey().count("Counts");
    // need to override value serde to Long type
    counts.to(Serdes.String(), Serdes.Long(), "streams-wordcount-output");
    KafkaStreams streams = new KafkaStreams(builder, props);
    streams.start();
    // usually the stream application would be running forever,
    // in this example we just let it run for some time and stop since the input data is finite.
    Thread.sleep(5000L);
    streams.close();
}

Example 17 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class StreamPartitionAssignorTest method shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks.

@Test
public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks() {
    final String applicationId = "application-id";
    final KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId(applicationId);
    KStream<Object, Object> stream1 = builder.stream("topic1").selectKey(new KeyValueMapper<Object, Object, Object>() {

        @Override
        public Object apply(Object key, Object value) {
            return null;
        }
    }).groupByKey().count("count").toStream().map(new KeyValueMapper<Object, Long, KeyValue<Object, Object>>() {

        @Override
        public KeyValue<Object, Object> apply(Object key, Long value) {
            return null;
        }
    });
    builder.stream("unknownTopic").selectKey(new KeyValueMapper<Object, Object, Object>() {

        @Override
        public Object apply(Object key, Object value) {
            return null;
        }
    }).join(stream1, new ValueJoiner() {

        @Override
        public Object apply(Object value1, Object value2) {
            return null;
        }
    }, JoinWindows.of(0));
    final UUID uuid = UUID.randomUUID();
    final String client = "client1";
    final StreamThread streamThread = new StreamThread(builder, config, mockClientSupplier, applicationId, client, uuid, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(streamThread, applicationId, client));
    final MockInternalTopicManager mockInternalTopicManager = new MockInternalTopicManager(streamThread.config, mockClientSupplier.restoreConsumer);
    partitionAssignor.setInternalTopicManager(mockInternalTopicManager);
    final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    final Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put(client, new PartitionAssignor.Subscription(Collections.singletonList("unknownTopic"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
    final Map<String, PartitionAssignor.Assignment> assignment = partitionAssignor.assign(metadata, subscriptions);
    final Map<String, Integer> expectedCreatedInternalTopics = new HashMap<>();
    expectedCreatedInternalTopics.put(applicationId + "-count-repartition", 3);
    expectedCreatedInternalTopics.put(applicationId + "-count-changelog", 3);
    assertThat(mockInternalTopicManager.readyTopics, equalTo(expectedCreatedInternalTopics));
    final List<TopicPartition> expectedAssignment = Arrays.asList(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2), new TopicPartition(applicationId + "-count-repartition", 0), new TopicPartition(applicationId + "-count-repartition", 1), new TopicPartition(applicationId + "-count-repartition", 2));
    assertThat(new HashSet(assignment.get(client).partitions()), equalTo(new HashSet(expectedAssignment)));
}

Example 18 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KTableAggregateTest method testRemoveOldBeforeAddNew.

@Test
public void testRemoveOldBeforeAddNew() throws IOException {
    final KStreamBuilder builder = new KStreamBuilder();
    final String input = "count-test-input";
    final MockProcessorSupplier<String, String> proc = new MockProcessorSupplier<>();
    builder.table(Serdes.String(), Serdes.String(), input, "anyStoreName").groupBy(new KeyValueMapper<String, String, KeyValue<String, String>>() {

        @Override
        public KeyValue<String, String> apply(String key, String value) {
            return KeyValue.pair(String.valueOf(key.charAt(0)), String.valueOf(key.charAt(1)));
        }
    }, stringSerde, stringSerde).aggregate(new Initializer<String>() {

        @Override
        public String apply() {
            return "";
        }
    }, new Aggregator<String, String, String>() {

        @Override
        public String apply(String aggKey, String value, String aggregate) {
            return aggregate + value;
        }
    }, new Aggregator<String, String, String>() {

        @Override
        public String apply(String key, String value, String aggregate) {
            return aggregate.replaceAll(value, "");
        }
    }, Serdes.String(), "someStore").toStream().process(proc);
    driver = new KStreamTestDriver(builder, stateDir);
    driver.process(input, "11", "A");
    driver.flushState();
    driver.process(input, "12", "B");
    driver.flushState();
    driver.process(input, "11", null);
    driver.flushState();
    driver.process(input, "12", "C");
    driver.flushState();
    assertEquals(Utils.mkList("1:1", "1:12", "1:2", "1:2"), proc.processed);
}

Example 19 with KeyValueMapper

use of org.apache.kafka.streams.kstream.KeyValueMapper in project kafka by apache.

the class KTableMapKeysTest method testMapKeysConvertingToStream.

@Test
public void testMapKeysConvertingToStream() {
    final KStreamBuilder builder = new KStreamBuilder();
    String topic1 = "topic_map_keys";
    KTable<Integer, String> table1 = builder.table(integerSerde, stringSerde, topic1, "anyStoreName");
    final Map<Integer, String> keyMap = new HashMap<>();
    keyMap.put(1, "ONE");
    keyMap.put(2, "TWO");
    keyMap.put(3, "THREE");
    KeyValueMapper<Integer, String, String> keyMapper = new KeyValueMapper<Integer, String, String>() {

        @Override
        public String apply(Integer key, String value) {
            return keyMap.get(key);
        }
    };
    KStream<String, String> convertedStream = table1.toStream(keyMapper);
    final String[] expected = new String[] { "ONE:V_ONE", "TWO:V_TWO", "THREE:V_THREE" };
    final int[] originalKeys = new int[] { 1, 2, 3 };
    final String[] values = new String[] { "V_ONE", "V_TWO", "V_THREE" };
    MockProcessorSupplier<String, String> processor = new MockProcessorSupplier<>();
    convertedStream.process(processor);
    driver = new KStreamTestDriver(builder, stateDir);
    for (int i = 0; i < originalKeys.length; i++) {
        driver.process(topic1, originalKeys[i], values[i]);
    }
    driver.flushState();
    assertEquals(3, processor.processed.size());
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.processed.get(i));
    }
}