Examples with KeyValue org.apache.kafka.streams.KeyValue used on opensource projects

Example 26 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class QueryableStateIntegrationTest method shouldBeAbleToQueryMapValuesState.

@Test
public void shouldBeAbleToQueryMapValuesState() throws Exception {
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    final StreamsBuilder builder = new StreamsBuilder();
    final String[] keys = { "hello", "goodbye", "welcome", "go", "kafka" };
    final Set<KeyValue<String, String>> batch1 = new HashSet<>(Arrays.asList(new KeyValue<>(keys[0], "1"), new KeyValue<>(keys[1], "1"), new KeyValue<>(keys[2], "3"), new KeyValue<>(keys[3], "5"), new KeyValue<>(keys[4], "2")));
    IntegrationTestUtils.produceKeyValuesSynchronously(streamOne, batch1, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), mockTime);
    final KTable<String, String> t1 = builder.table(streamOne);
    final KTable<String, Long> t2 = t1.mapValues(new ValueMapper<String, Long>() {

        @Override
        public Long apply(final String value) {
            return Long.valueOf(value);
        }
    }, Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("queryMapValues").withValueSerde(Serdes.Long()));
    t2.toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
    kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
    kafkaStreams.start();
    waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
    final ReadOnlyKeyValueStore<String, Long> myMapStore = kafkaStreams.store("queryMapValues", QueryableStoreTypes.<String, Long>keyValueStore());
    for (final KeyValue<String, String> batchEntry : batch1) {
        assertEquals(myMapStore.get(batchEntry.key), Long.valueOf(batchEntry.value));
    }
}

Example 27 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class RegexSourceIntegrationTest method testShouldReadFromRegexAndNamedTopics.

@Test
public void testShouldReadFromRegexAndNamedTopics() throws Exception {
    final String topic1TestMessage = "topic-1 test";
    final String topic2TestMessage = "topic-2 test";
    final String topicATestMessage = "topic-A test";
    final String topicCTestMessage = "topic-C test";
    final String topicYTestMessage = "topic-Y test";
    final String topicZTestMessage = "topic-Z test";
    final Serde<String> stringSerde = Serdes.String();
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> pattern1Stream = builder.stream(Pattern.compile("topic-\\d"));
    final KStream<String, String> pattern2Stream = builder.stream(Pattern.compile("topic-[A-D]"));
    final KStream<String, String> namedTopicsStream = builder.stream(Arrays.asList(TOPIC_Y, TOPIC_Z));
    pattern1Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    pattern2Stream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    namedTopicsStream.to(stringSerde, stringSerde, DEFAULT_OUTPUT_TOPIC);
    streams = new KafkaStreams(builder.build(), streamsConfiguration);
    streams.start();
    final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_1, Arrays.asList(topic1TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_2, Arrays.asList(topic2TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_A, Arrays.asList(topicATestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_C, Arrays.asList(topicCTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_Y, Arrays.asList(topicYTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(TOPIC_Z, Arrays.asList(topicZTestMessage), producerConfig, mockTime);
    final Properties consumerConfig = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, StringDeserializer.class);
    final List<String> expectedReceivedValues = Arrays.asList(topicATestMessage, topic1TestMessage, topic2TestMessage, topicCTestMessage, topicYTestMessage, topicZTestMessage);
    final List<KeyValue<String, String>> receivedKeyValues = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(consumerConfig, DEFAULT_OUTPUT_TOPIC, 6);
    final List<String> actualValues = new ArrayList<>(6);
    for (final KeyValue<String, String> receivedKeyValue : receivedKeyValues) {
        actualValues.add(receivedKeyValue.value);
    }
    Collections.sort(actualValues);
    Collections.sort(expectedReceivedValues);
    assertThat(actualValues, equalTo(expectedReceivedValues));
}

Example 28 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class KTableForeachTest method testForeach.

@Test
public void testForeach() {
    // Given
    List<KeyValue<Integer, String>> inputRecords = Arrays.asList(new KeyValue<>(0, "zero"), new KeyValue<>(1, "one"), new KeyValue<>(2, "two"), new KeyValue<>(3, "three"));
    List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(0, "ZERO"), new KeyValue<>(2, "ONE"), new KeyValue<>(4, "TWO"), new KeyValue<>(6, "THREE"));
    final List<KeyValue<Integer, String>> actualRecords = new ArrayList<>();
    ForeachAction<Integer, String> action = new ForeachAction<Integer, String>() {

        @Override
        public void apply(Integer key, String value) {
            actualRecords.add(new KeyValue<>(key * 2, value.toUpperCase(Locale.ROOT)));
        }
    };
    // When
    StreamsBuilder builder = new StreamsBuilder();
    KTable<Integer, String> table = builder.table(topicName, Consumed.with(intSerde, stringSerde), Materialized.<Integer, String, KeyValueStore<Bytes, byte[]>>as(topicName).withKeySerde(intSerde).withValueSerde(stringSerde));
    table.foreach(action);
    // Then
    driver.setUp(builder, stateDir);
    for (KeyValue<Integer, String> record : inputRecords) {
        driver.process(topicName, record.key, record.value);
    }
    driver.flushState();
    assertEquals(expectedRecords.size(), actualRecords.size());
    for (int i = 0; i < expectedRecords.size(); i++) {
        KeyValue<Integer, String> expectedRecord = expectedRecords.get(i);
        KeyValue<Integer, String> actualRecord = actualRecords.get(i);
        assertEquals(expectedRecord, actualRecord);
    }
}

Example 29 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class KStreamFlatMapTest method testFlatMap.

@Test
public void testFlatMap() {
    StreamsBuilder builder = new StreamsBuilder();
    KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>> mapper = new KeyValueMapper<Number, Object, Iterable<KeyValue<String, String>>>() {

        @Override
        public Iterable<KeyValue<String, String>> apply(Number key, Object value) {
            ArrayList<KeyValue<String, String>> result = new ArrayList<>();
            for (int i = 0; i < key.intValue(); i++) {
                result.add(KeyValue.pair(Integer.toString(key.intValue() * 10 + i), value.toString()));
            }
            return result;
        }
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    KStream<Integer, String> stream;
    MockProcessorSupplier<String, String> processor;
    processor = new MockProcessorSupplier<>();
    stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
    stream.flatMap(mapper).process(processor);
    driver.setUp(builder);
    for (int expectedKey : expectedKeys) {
        driver.process(topicName, expectedKey, "V" + expectedKey);
    }
    assertEquals(6, processor.processed.size());
    String[] expected = { "10:V1", "20:V2", "21:V2", "30:V3", "31:V3", "32:V3" };
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.processed.get(i));
    }
}

Example 30 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class KStreamForeachTest method testForeach.

@Test
public void testForeach() {
    // Given
    List<KeyValue<Integer, String>> inputRecords = Arrays.asList(new KeyValue<>(0, "zero"), new KeyValue<>(1, "one"), new KeyValue<>(2, "two"), new KeyValue<>(3, "three"));
    List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(0, "ZERO"), new KeyValue<>(2, "ONE"), new KeyValue<>(4, "TWO"), new KeyValue<>(6, "THREE"));
    final List<KeyValue<Integer, String>> actualRecords = new ArrayList<>();
    ForeachAction<Integer, String> action = new ForeachAction<Integer, String>() {

        @Override
        public void apply(Integer key, String value) {
            actualRecords.add(new KeyValue<>(key * 2, value.toUpperCase(Locale.ROOT)));
        }
    };
    // When
    StreamsBuilder builder = new StreamsBuilder();
    KStream<Integer, String> stream = builder.stream(topicName, Consumed.with(intSerde, stringSerde));
    stream.foreach(action);
    // Then
    driver.setUp(builder);
    for (KeyValue<Integer, String> record : inputRecords) {
        driver.process(topicName, record.key, record.value);
    }
    assertEquals(expectedRecords.size(), actualRecords.size());
    for (int i = 0; i < expectedRecords.size(); i++) {
        KeyValue<Integer, String> expectedRecord = expectedRecords.get(i);
        KeyValue<Integer, String> actualRecord = actualRecords.get(i);
        assertEquals(expectedRecord, actualRecord);
    }
}