Examples with KeyValue org.apache.kafka.streams.KeyValue used on opensource projects

Example 21 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class KStreamAggregationIntegrationTest method shouldAggregateWindowed.

@Test
public void shouldAggregateWindowed() throws Exception {
    final long firstTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstTimestamp);
    final long secondTimestamp = mockTime.milliseconds();
    produceMessages(secondTimestamp);
    produceMessages(secondTimestamp);
    groupedStream.windowedBy(TimeWindows.of(500L)).aggregate(initializer, aggregator, Materialized.<String, Integer, WindowStore<Bytes, byte[]>>with(null, Serdes.Integer())).toStream(new KeyValueMapper<Windowed<String>, Integer, String>() {

        @Override
        public String apply(final Windowed<String> windowedKey, final Integer value) {
            return windowedKey.key() + "@" + windowedKey.window().start();
        }
    }).to(outputTopic, Produced.with(Serdes.String(), Serdes.Integer()));
    startStreams();
    final List<KeyValue<String, Integer>> windowedMessages = receiveMessages(new StringDeserializer(), new IntegerDeserializer(), 15);
    final Comparator<KeyValue<String, Integer>> comparator = new Comparator<KeyValue<String, Integer>>() {

        @Override
        public int compare(final KeyValue<String, Integer> o1, final KeyValue<String, Integer> o2) {
            return KStreamAggregationIntegrationTest.compare(o1, o2);
        }
    };
    Collections.sort(windowedMessages, comparator);
    final long firstWindow = firstTimestamp / 500 * 500;
    final long secondWindow = secondTimestamp / 500 * 500;
    assertThat(windowedMessages, is(Arrays.asList(new KeyValue<>("A@" + firstWindow, 1), new KeyValue<>("A@" + secondWindow, 1), new KeyValue<>("A@" + secondWindow, 2), new KeyValue<>("B@" + firstWindow, 1), new KeyValue<>("B@" + secondWindow, 1), new KeyValue<>("B@" + secondWindow, 2), new KeyValue<>("C@" + firstWindow, 1), new KeyValue<>("C@" + secondWindow, 1), new KeyValue<>("C@" + secondWindow, 2), new KeyValue<>("D@" + firstWindow, 1), new KeyValue<>("D@" + secondWindow, 1), new KeyValue<>("D@" + secondWindow, 2), new KeyValue<>("E@" + firstWindow, 1), new KeyValue<>("E@" + secondWindow, 1), new KeyValue<>("E@" + secondWindow, 2))));
}

Example 22 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class KStreamsFineGrainedAutoResetIntegrationTest method shouldOnlyReadForEarliest.

private void shouldOnlyReadForEarliest(final String topicSuffix, final String topic1, final String topic2, final String topicA, final String topicC, final String topicY, final String topicZ, final String outputTopic, final List<String> expectedReceivedValues) throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> pattern1Stream = builder.stream(Pattern.compile("topic-\\d" + topicSuffix), Consumed.<String, String>with(Topology.AutoOffsetReset.EARLIEST));
    final KStream<String, String> pattern2Stream = builder.stream(Pattern.compile("topic-[A-D]" + topicSuffix), Consumed.<String, String>with(Topology.AutoOffsetReset.LATEST));
    final KStream<String, String> namedTopicsStream = builder.stream(Arrays.asList(topicY, topicZ));
    pattern1Stream.to(stringSerde, stringSerde, outputTopic);
    pattern2Stream.to(stringSerde, stringSerde, outputTopic);
    namedTopicsStream.to(stringSerde, stringSerde, outputTopic);
    final Properties producerConfig = TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class);
    IntegrationTestUtils.produceValuesSynchronously(topic1, Collections.singletonList(topic1TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(topic2, Collections.singletonList(topic2TestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(topicA, Collections.singletonList(topicATestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(topicC, Collections.singletonList(topicCTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(topicY, Collections.singletonList(topicYTestMessage), producerConfig, mockTime);
    IntegrationTestUtils.produceValuesSynchronously(topicZ, Collections.singletonList(topicZTestMessage), producerConfig, mockTime);
    final Properties consumerConfig = TestUtils.consumerConfig(CLUSTER.bootstrapServers(), StringDeserializer.class, StringDeserializer.class);
    final KafkaStreams streams = new KafkaStreams(builder.build(), streamsConfiguration);
    streams.start();
    final List<KeyValue<String, String>> receivedKeyValues = IntegrationTestUtils.waitUntilMinKeyValueRecordsReceived(consumerConfig, outputTopic, expectedReceivedValues.size());
    final List<String> actualValues = new ArrayList<>(expectedReceivedValues.size());
    for (final KeyValue<String, String> receivedKeyValue : receivedKeyValues) {
        actualValues.add(receivedKeyValue.value);
    }
    streams.close();
    Collections.sort(actualValues);
    Collections.sort(expectedReceivedValues);
    assertThat(actualValues, equalTo(expectedReceivedValues));
}

Example 23 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class QueryableStateIntegrationTest method verifyCanQueryState.

private void verifyCanQueryState(final int cacheSizeBytes) throws Exception {
    streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, cacheSizeBytes);
    final StreamsBuilder builder = new StreamsBuilder();
    final String[] keys = { "hello", "goodbye", "welcome", "go", "kafka" };
    final Set<KeyValue<String, String>> batch1 = new TreeSet<>(stringComparator);
    batch1.addAll(Arrays.asList(new KeyValue<>(keys[0], "hello"), new KeyValue<>(keys[1], "goodbye"), new KeyValue<>(keys[2], "welcome"), new KeyValue<>(keys[3], "go"), new KeyValue<>(keys[4], "kafka")));
    final Set<KeyValue<String, Long>> expectedCount = new TreeSet<>(stringLongComparator);
    for (final String key : keys) {
        expectedCount.add(new KeyValue<>(key, 1L));
    }
    IntegrationTestUtils.produceKeyValuesSynchronously(streamOne, batch1, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, StringSerializer.class, new Properties()), mockTime);
    final KStream<String, String> s1 = builder.stream(streamOne);
    // Non Windowed
    final String storeName = "my-count";
    s1.groupByKey().count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(storeName)).toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
    final String windowStoreName = "windowed-count";
    s1.groupByKey().windowedBy(TimeWindows.of(WINDOW_SIZE)).count(Materialized.<String, Long, WindowStore<Bytes, byte[]>>as(windowStoreName));
    kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
    kafkaStreams.start();
    waitUntilAtLeastNumRecordProcessed(outputTopic, 1);
    final ReadOnlyKeyValueStore<String, Long> myCount = kafkaStreams.store(storeName, QueryableStoreTypes.<String, Long>keyValueStore());
    final ReadOnlyWindowStore<String, Long> windowStore = kafkaStreams.store(windowStoreName, QueryableStoreTypes.<String, Long>windowStore());
    verifyCanGetByKey(keys, expectedCount, expectedCount, windowStore, myCount);
    verifyRangeAndAll(expectedCount, myCount);
}

Example 24 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class QueryableStateIntegrationTest method shouldBeAbleToQueryFilterState.

@Test
public void shouldBeAbleToQueryFilterState() throws Exception {
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass());
    final StreamsBuilder builder = new StreamsBuilder();
    final String[] keys = { "hello", "goodbye", "welcome", "go", "kafka" };
    final Set<KeyValue<String, Long>> batch1 = new HashSet<>(Arrays.asList(new KeyValue<>(keys[0], 1L), new KeyValue<>(keys[1], 1L), new KeyValue<>(keys[2], 3L), new KeyValue<>(keys[3], 5L), new KeyValue<>(keys[4], 2L)));
    final Set<KeyValue<String, Long>> expectedBatch1 = new HashSet<>(Collections.singleton(new KeyValue<>(keys[4], 2L)));
    IntegrationTestUtils.produceKeyValuesSynchronously(streamOne, batch1, TestUtils.producerConfig(CLUSTER.bootstrapServers(), StringSerializer.class, LongSerializer.class, new Properties()), mockTime);
    final Predicate<String, Long> filterPredicate = new Predicate<String, Long>() {

        @Override
        public boolean test(final String key, final Long value) {
            return key.contains("kafka");
        }
    };
    final KTable<String, Long> t1 = builder.table(streamOne);
    final KTable<String, Long> t2 = t1.filter(filterPredicate, Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("queryFilter"));
    t1.filterNot(filterPredicate, Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("queryFilterNot"));
    t2.toStream().to(outputTopic);
    kafkaStreams = new KafkaStreams(builder.build(), streamsConfiguration);
    kafkaStreams.start();
    waitUntilAtLeastNumRecordProcessed(outputTopic, 2);
    final ReadOnlyKeyValueStore<String, Long> myFilterStore = kafkaStreams.store("queryFilter", QueryableStoreTypes.<String, Long>keyValueStore());
    final ReadOnlyKeyValueStore<String, Long> myFilterNotStore = kafkaStreams.store("queryFilterNot", QueryableStoreTypes.<String, Long>keyValueStore());
    for (final KeyValue<String, Long> expectedEntry : expectedBatch1) {
        assertEquals(myFilterStore.get(expectedEntry.key), expectedEntry.value);
    }
    for (final KeyValue<String, Long> batchEntry : batch1) {
        if (!expectedBatch1.contains(batchEntry)) {
            assertNull(myFilterStore.get(batchEntry.key));
        }
    }
    for (final KeyValue<String, Long> expectedEntry : expectedBatch1) {
        assertNull(myFilterNotStore.get(expectedEntry.key));
    }
    for (final KeyValue<String, Long> batchEntry : batch1) {
        if (!expectedBatch1.contains(batchEntry)) {
            assertEquals(myFilterNotStore.get(batchEntry.key), batchEntry.value);
        }
    }
}

Example 25 with KeyValue

use of org.apache.kafka.streams.KeyValue in project apache-kafka-on-k8s by banzaicloud.

the class QueryableStateIntegrationTest method before.

@Before
public void before() throws Exception {
    testNo++;
    createTopics();
    streamsConfiguration = new Properties();
    final String applicationId = "queryable-state-" + testNo;
    streamsConfiguration.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
    streamsConfiguration.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    streamsConfiguration.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    streamsConfiguration.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory("qs-test").getPath());
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 100);
    // override this to make the rebalances happen quickly
    streamsConfiguration.put(IntegrationTestUtils.INTERNAL_LEAVE_GROUP_ON_CLOSE, true);
    stringComparator = new Comparator<KeyValue<String, String>>() {

        @Override
        public int compare(final KeyValue<String, String> o1, final KeyValue<String, String> o2) {
            return o1.key.compareTo(o2.key);
        }
    };
    stringLongComparator = new Comparator<KeyValue<String, Long>>() {

        @Override
        public int compare(final KeyValue<String, Long> o1, final KeyValue<String, Long> o2) {
            return o1.key.compareTo(o2.key);
        }
    };
    inputValues = getInputValues();
    inputValuesKeys = new HashSet<>();
    for (final String sentence : inputValues) {
        final String[] words = sentence.split("\\W+");
        numberOfWordsPerIteration += words.length;
        Collections.addAll(inputValuesKeys, words);
    }
}