Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 36 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldAggregateWindowed.

@SuppressWarnings("deprecation")
@Test
public void shouldAggregateWindowed() throws Exception {
    final long firstTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstTimestamp);
    final long secondTimestamp = mockTime.milliseconds();
    produceMessages(secondTimestamp);
    produceMessages(secondTimestamp);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, 500L);
    // noinspection deprecation
    groupedStream.windowedBy(TimeWindows.of(ofMillis(500L))).aggregate(initializer, aggregator, Materialized.with(null, Serdes.Integer())).toStream().to(outputTopic, Produced.with(windowedSerde, Serdes.Integer()));
    startStreams();
    final List<KeyValueTimestamp<Windowed<String>, Integer>> windowedMessages = receiveMessagesWithTimestamp(new TimeWindowedDeserializer<>(new StringDeserializer(), 500L), new IntegerDeserializer(), String.class, 15);
    // read from ConsoleConsumer
    final String resultFromConsoleConsumer = readWindowedKeyedMessagesViaConsoleConsumer(new TimeWindowedDeserializer<String>(), new IntegerDeserializer(), String.class, 15, true);
    final Comparator<KeyValueTimestamp<Windowed<String>, Integer>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, Integer> o) -> o.key().key()).thenComparingInt(KeyValueTimestamp::value);
    windowedMessages.sort(comparator);
    final long firstWindowStart = firstTimestamp / 500 * 500;
    final long firstWindowEnd = firstWindowStart + 500;
    final long secondWindowStart = secondTimestamp / 500 * 500;
    final long secondWindowEnd = secondWindowStart + 500;
    final List<KeyValueTimestamp<Windowed<String>, Integer>> expectResult = Arrays.asList(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstWindowStart, firstWindowEnd)), 1, firstTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondWindowStart, secondWindowEnd)), 1, secondTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondWindowStart, secondWindowEnd)), 2, secondTimestamp));
    assertThat(windowedMessages, is(expectResult));
    final Set<String> expectResultString = new HashSet<>(expectResult.size());
    for (final KeyValueTimestamp<Windowed<String>, Integer> eachRecord : expectResult) {
        expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value());
    }
    // check every message is contained in the expect result
    final String[] allRecords = resultFromConsoleConsumer.split("\n");
    for (final String record : allRecords) {
        assertTrue(expectResultString.contains(record));
    }
}

Example 37 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldCountHelper.

private void shouldCountHelper() throws Exception {
    startStreams();
    produceMessages(mockTime.milliseconds());
    final List<KeyValueTimestamp<String, Long>> results = receiveMessages(new StringDeserializer(), new LongDeserializer(), 10);
    results.sort(KStreamAggregationIntegrationTest::compare);
    assertThat(results, is(Arrays.asList(new KeyValueTimestamp("A", 1L, mockTime.milliseconds()), new KeyValueTimestamp("A", 2L, mockTime.milliseconds()), new KeyValueTimestamp("B", 1L, mockTime.milliseconds()), new KeyValueTimestamp("B", 2L, mockTime.milliseconds()), new KeyValueTimestamp("C", 1L, mockTime.milliseconds()), new KeyValueTimestamp("C", 2L, mockTime.milliseconds()), new KeyValueTimestamp("D", 1L, mockTime.milliseconds()), new KeyValueTimestamp("D", 2L, mockTime.milliseconds()), new KeyValueTimestamp("E", 1L, mockTime.milliseconds()), new KeyValueTimestamp("E", 2L, mockTime.milliseconds()))));
}

Example 38 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationDedupIntegrationTest method shouldReduceWindowed.

@Test
public void shouldReduceWindowed() throws Exception {
    final long firstBatchTimestamp = System.currentTimeMillis() - 1000;
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = System.currentTimeMillis();
    produceMessages(secondBatchTimestamp);
    produceMessages(secondBatchTimestamp);
    groupedStream.windowedBy(TimeWindows.of(ofMillis(500L))).reduce(reducer, Materialized.as("reduce-time-windows")).toStream((windowedKey, value) -> windowedKey.key() + "@" + windowedKey.window().start()).to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    startStreams();
    final long firstBatchWindow = firstBatchTimestamp / 500 * 500;
    final long secondBatchWindow = secondBatchTimestamp / 500 * 500;
    validateReceivedMessages(new StringDeserializer(), new StringDeserializer(), Arrays.asList(new KeyValueTimestamp<>("A@" + firstBatchWindow, "A", firstBatchTimestamp), new KeyValueTimestamp<>("A@" + secondBatchWindow, "A:A", secondBatchTimestamp), new KeyValueTimestamp<>("B@" + firstBatchWindow, "B", firstBatchTimestamp), new KeyValueTimestamp<>("B@" + secondBatchWindow, "B:B", secondBatchTimestamp), new KeyValueTimestamp<>("C@" + firstBatchWindow, "C", firstBatchTimestamp), new KeyValueTimestamp<>("C@" + secondBatchWindow, "C:C", secondBatchTimestamp), new KeyValueTimestamp<>("D@" + firstBatchWindow, "D", firstBatchTimestamp), new KeyValueTimestamp<>("D@" + secondBatchWindow, "D:D", secondBatchTimestamp), new KeyValueTimestamp<>("E@" + firstBatchWindow, "E", firstBatchTimestamp), new KeyValueTimestamp<>("E@" + secondBatchWindow, "E:E", secondBatchTimestamp)));
}

Example 39 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldReduceWindowed.

@SuppressWarnings("deprecation")
@Test
public void shouldReduceWindowed() throws Exception {
    final long firstBatchTimestamp = mockTime.milliseconds();
    mockTime.sleep(1000);
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = mockTime.milliseconds();
    produceMessages(secondBatchTimestamp);
    produceMessages(secondBatchTimestamp);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, 500L);
    // noinspection deprecation
    groupedStream.windowedBy(TimeWindows.of(ofMillis(500L))).reduce(reducer).toStream().to(outputTopic, Produced.with(windowedSerde, Serdes.String()));
    startStreams();
    final List<KeyValueTimestamp<Windowed<String>, String>> windowedOutput = receiveMessages(new TimeWindowedDeserializer<>(), new StringDeserializer(), String.class, 15);
    // read from ConsoleConsumer
    final String resultFromConsoleConsumer = readWindowedKeyedMessagesViaConsoleConsumer(new TimeWindowedDeserializer<String>(), new StringDeserializer(), String.class, 15, true);
    final Comparator<KeyValueTimestamp<Windowed<String>, String>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, String> o) -> o.key().key()).thenComparing(KeyValueTimestamp::value);
    windowedOutput.sort(comparator);
    final long firstBatchWindowStart = firstBatchTimestamp / 500 * 500;
    final long firstBatchWindowEnd = firstBatchWindowStart + 500;
    final long secondBatchWindowStart = secondBatchTimestamp / 500 * 500;
    final long secondBatchWindowEnd = secondBatchWindowStart + 500;
    final List<KeyValueTimestamp<Windowed<String>, String>> expectResult = Arrays.asList(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchWindowStart, firstBatchWindowEnd)), "A", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "A", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "A:A", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchWindowStart, firstBatchWindowEnd)), "B", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "B", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "B:B", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchWindowStart, firstBatchWindowEnd)), "C", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "C", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "C:C", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchWindowStart, firstBatchWindowEnd)), "D", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "D", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "D:D", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchWindowStart, firstBatchWindowEnd)), "E", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "E", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchWindowStart, secondBatchWindowEnd)), "E:E", secondBatchTimestamp));
    assertThat(windowedOutput, is(expectResult));
    final Set<String> expectResultString = new HashSet<>(expectResult.size());
    for (final KeyValueTimestamp<Windowed<String>, String> eachRecord : expectResult) {
        expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value());
    }
    // check every message is contained in the expect result
    final String[] allRecords = resultFromConsoleConsumer.split("\n");
    for (final String record : allRecords) {
        assertTrue(expectResultString.contains(record));
    }
}

Example 40 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldAggregate.

@Test
public void shouldAggregate() throws Exception {
    produceMessages(mockTime.milliseconds());
    groupedStream.aggregate(initializer, aggregator, Materialized.as("aggregate-by-selected-key")).toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.Integer()));
    startStreams();
    produceMessages(mockTime.milliseconds());
    final List<KeyValueTimestamp<String, Integer>> results = receiveMessages(new StringDeserializer(), new IntegerDeserializer(), 10);
    results.sort(KStreamAggregationIntegrationTest::compare);
    assertThat(results, is(Arrays.asList(new KeyValueTimestamp("A", 1, mockTime.milliseconds()), new KeyValueTimestamp("A", 2, mockTime.milliseconds()), new KeyValueTimestamp("B", 1, mockTime.milliseconds()), new KeyValueTimestamp("B", 2, mockTime.milliseconds()), new KeyValueTimestamp("C", 1, mockTime.milliseconds()), new KeyValueTimestamp("C", 2, mockTime.milliseconds()), new KeyValueTimestamp("D", 1, mockTime.milliseconds()), new KeyValueTimestamp("D", 2, mockTime.milliseconds()), new KeyValueTimestamp("E", 1, mockTime.milliseconds()), new KeyValueTimestamp("E", 2, mockTime.milliseconds()))));
}