Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 41 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldReduce.

@Test
public void shouldReduce() throws Exception {
    produceMessages(mockTime.milliseconds());
    groupedStream.reduce(reducer, Materialized.as("reduce-by-key")).toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
    startStreams();
    produceMessages(mockTime.milliseconds());
    final List<KeyValueTimestamp<String, String>> results = receiveMessages(new StringDeserializer(), new StringDeserializer(), 10);
    results.sort(KStreamAggregationIntegrationTest::compare);
    assertThat(results, is(Arrays.asList(new KeyValueTimestamp("A", "A", mockTime.milliseconds()), new KeyValueTimestamp("A", "A:A", mockTime.milliseconds()), new KeyValueTimestamp("B", "B", mockTime.milliseconds()), new KeyValueTimestamp("B", "B:B", mockTime.milliseconds()), new KeyValueTimestamp("C", "C", mockTime.milliseconds()), new KeyValueTimestamp("C", "C:C", mockTime.milliseconds()), new KeyValueTimestamp("D", "D", mockTime.milliseconds()), new KeyValueTimestamp("D", "D:D", mockTime.milliseconds()), new KeyValueTimestamp("E", "E", mockTime.milliseconds()), new KeyValueTimestamp("E", "E:E", mockTime.milliseconds()))));
}

Example 42 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamSessionWindowAggregateProcessorTest method setup.

private void setup(final boolean enableCache) {
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "test", StreamsConfig.METRICS_LATEST, new MockTime());
    context = new InternalMockProcessorContext<Windowed<String>, Change<Long>>(TestUtils.tempDirectory(), Serdes.String(), Serdes.String(), streamsMetrics, new StreamsConfig(StreamsTestUtils.getStreamsConfig()), MockRecordCollector::new, new ThreadCache(new LogContext("testCache "), 100000, streamsMetrics), Time.SYSTEM) {

        @Override
        public <K extends Windowed<String>, V extends Change<Long>> void forward(final Record<K, V> record) {
            results.add(new KeyValueTimestamp<>(record.key(), record.value(), record.timestamp()));
        }
    };
    // Set initial timestamp for CachingSessionStore to prepare entry from as default
    // InternalMockProcessorContext#timestamp returns -1.
    context.setTime(0L);
    TaskMetrics.droppedRecordsSensor(threadId, context.taskId().toString(), streamsMetrics);
    initStore(enableCache);
    processor.init(context);
}

Example 43 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamSlidingWindowAggregateTest method testAggregateLargeInput.

@Test
public void testAggregateLargeInput() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final WindowBytesStoreSupplier storeSupplier = inOrderIterator ? new InOrderMemoryWindowStoreSupplier("InOrder", 50000L, 10L, false) : Stores.inMemoryWindowStore("Reverse", Duration.ofMillis(50000), Duration.ofMillis(10), false);
    final KTable<Windowed<String>, String> table2 = builder.stream(topic1, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(SlidingWindows.ofTimeDifferenceAndGrace(ofMillis(10), ofMillis(50))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.as(storeSupplier));
    final MockApiProcessorSupplier<Windowed<String>, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table2.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        inputTopic1.pipeInput("A", "1", 10L);
        inputTopic1.pipeInput("A", "2", 20L);
        inputTopic1.pipeInput("A", "3", 22L);
        inputTopic1.pipeInput("A", "4", 15L);
        inputTopic1.pipeInput("B", "1", 12L);
        inputTopic1.pipeInput("B", "2", 13L);
        inputTopic1.pipeInput("B", "3", 18L);
        inputTopic1.pipeInput("B", "4", 19L);
        inputTopic1.pipeInput("B", "5", 25L);
        inputTopic1.pipeInput("B", "6", 14L);
        inputTopic1.pipeInput("C", "1", 11L);
        inputTopic1.pipeInput("C", "2", 15L);
        inputTopic1.pipeInput("C", "3", 16L);
        inputTopic1.pipeInput("C", "4", 21);
        inputTopic1.pipeInput("C", "5", 23L);
        inputTopic1.pipeInput("D", "4", 11L);
        inputTopic1.pipeInput("D", "2", 12L);
        inputTopic1.pipeInput("D", "3", 29L);
        inputTopic1.pipeInput("D", "5", 16L);
    }
    final Comparator<KeyValueTimestamp<Windowed<String>, String>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, String> o) -> o.key().key()).thenComparing((KeyValueTimestamp<Windowed<String>, String> o) -> o.key().window().start());
    final ArrayList<KeyValueTimestamp<Windowed<String>, String>> actual = supplier.theCapturedProcessor().processed();
    actual.sort(comparator);
    assertEquals(asList(// FINAL WINDOW: A@10 left window created when A@10 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1", 10), // FINAL WINDOW: A@15 left window created when A@15 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+4", 15), // A@20 left window created when A@20 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(10, 20)), "0+1+2", 20), // FINAL WINDOW: A@20 left window updated when A@15 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(10, 20)), "0+1+2+4", 20), // A@10 right window created when A@20 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(11, 21)), "0+2", 20), // FINAL WINDOW: A@10 right window updated when A@15 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(11, 21)), "0+2+4", 20), // A@22 left window created when A@22 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(12, 22)), "0+2+3", 22), // FINAL WINDOW: A@22 left window updated when A@15 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(12, 22)), "0+2+3+4", 22), // FINAL WINDOW: A@15 right window created when A@15 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(16, 26)), "0+2+3", 22), // FINAL WINDOW: A@20 right window created when A@22 processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(21, 31)), "0+3", 22), // FINAL WINDOW: B@12 left window created when B@12 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(2, 12)), "0+1", 12), // FINAL WINDOW: B@13 left window created when B@13 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(3, 13)), "0+1+2", 13), // FINAL WINDOW: B@14 left window created when B@14 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(4, 14)), "0+1+2+6", 14), // B@18 left window created when B@18 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(8, 18)), "0+1+2+3", 18), // FINAL WINDOW: B@18 left window updated when B@14 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(8, 18)), "0+1+2+3+6", 18), // B@19 left window created when B@19 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(9, 19)), "0+1+2+3+4", 19), // FINAL WINDOW: B@19 left window updated when B@14 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(9, 19)), "0+1+2+3+4+6", 19), // B@12 right window created when B@13 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2", 13), // B@12 right window updated when B@18 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2+3", 18), // B@12 right window updated when B@19 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2+3+4", 19), // FINAL WINDOW: B@12 right window updated when B@14 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2+3+4+6", 19), // B@13 right window created when B@18 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(14, 24)), "0+3", 18), // B@13 right window updated when B@19 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(14, 24)), "0+3+4", 19), // FINAL WINDOW: B@13 right window updated when B@14 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(14, 24)), "0+3+4+6", 19), // FINAL WINDOW: B@25 left window created when B@25 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(15, 25)), "0+3+4+5", 25), // B@18 right window created when B@19 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(19, 29)), "0+4", 19), // FINAL WINDOW: B@18 right window updated when B@25 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(19, 29)), "0+4+5", 25), // FINAL WINDOW: B@19 right window updated when B@25 processed
    new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(20, 30)), "0+5", 25), // FINAL WINDOW: C@11 left window created when C@11 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(1, 11)), "0+1", 11), // FINAL WINDOW: C@15 left window created when C@15 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(5, 15)), "0+1+2", 15), // FINAL WINDOW: C@16 left window created when C@16 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(6, 16)), "0+1+2+3", 16), // FINAL WINDOW: C@21 left window created when C@21 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(11, 21)), "0+1+2+3+4", 21), // C@11 right window created when C@15 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(12, 22)), "0+2", 15), // C@11 right window updated when C@16 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(12, 22)), "0+2+3", 16), // FINAL WINDOW: C@11 right window updated when C@21 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(12, 22)), "0+2+3+4", 21), // FINAL WINDOW: C@23 left window created when C@23 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(13, 23)), "0+2+3+4+5", 23), // C@15 right window created when C@16 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(16, 26)), "0+3", 16), // C@15 right window updated when C@21 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(16, 26)), "0+3+4", 21), // FINAL WINDOW: C@15 right window updated when C@23 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(16, 26)), "0+3+4+5", 23), // C@16 right window created when C@21 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(17, 27)), "0+4", 21), // FINAL WINDOW: C@16 right window updated when C@23 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(17, 27)), "0+4+5", 23), // FINAL WINDOW: C@21 right window created when C@23 processed
    new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(22, 32)), "0+5", 23), // FINAL WINDOW: D@11 left window created when D@11 processed
    new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(1, 11)), "0+4", 11), // FINAL WINDOW: D@12 left window created when D@12 processed
    new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(2, 12)), "0+4+2", 12), // FINAL WINDOW: D@16 left window created when D@16 processed
    new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(6, 16)), "0+4+2+5", 16), // D@11 right window created when D@12 processed
    new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(12, 22)), "0+2", 12), // FINAL WINDOW: D@11 right window updated when D@16 processed
    new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(12, 22)), "0+2+5", 16), // FINAL WINDOW: D@12 right window created when D@16 processed
    new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(13, 23)), "0+5", 16), // FINAL WINDOW: D@29 left window created when D@29 processed
    new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(19, 29)), "0+3", 29)), actual);
}

Example 44 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamTransformTest method testTransformWithNewDriverAndPunctuator.

// Old PAPI. Needs to be migrated.
@SuppressWarnings("deprecation")
@Test
public void testTransformWithNewDriverAndPunctuator() {
    final StreamsBuilder builder = new StreamsBuilder();
    final TransformerSupplier<Number, Number, KeyValue<Integer, Integer>> transformerSupplier = () -> new Transformer<Number, Number, KeyValue<Integer, Integer>>() {

        private int total = 0;

        @Override
        public void init(final ProcessorContext context) {
            context.schedule(Duration.ofMillis(1), PunctuationType.WALL_CLOCK_TIME, timestamp -> context.forward(-1, (int) timestamp, To.all().withTimestamp(timestamp)));
        }

        @Override
        public KeyValue<Integer, Integer> transform(final Number key, final Number value) {
            total += value.intValue();
            return KeyValue.pair(key.intValue() * 2, total);
        }

        @Override
        public void close() {
        }
    };
    final int[] expectedKeys = { 1, 10, 100, 1000 };
    final MockProcessorSupplier<Integer, Integer> processor = new MockProcessorSupplier<>();
    final KStream<Integer, Integer> stream = builder.stream(TOPIC_NAME, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    stream.transform(transformerSupplier).process(processor);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, Instant.ofEpochMilli(0L))) {
        final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(TOPIC_NAME, new IntegerSerializer(), new IntegerSerializer());
        for (final int expectedKey : expectedKeys) {
            inputTopic.pipeInput(expectedKey, expectedKey * 10, 0L);
        }
        // This tick yields the "-1:2" result
        driver.advanceWallClockTime(Duration.ofMillis(2));
        // This tick further advances the clock to 3, which leads to the "-1:3" result
        driver.advanceWallClockTime(Duration.ofMillis(1));
    }
    assertEquals(6, processor.theCapturedProcessor().processed().size());
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(2, 10, 0), new KeyValueTimestamp<>(20, 110, 0), new KeyValueTimestamp<>(200, 1110, 0), new KeyValueTimestamp<>(2000, 11110, 0), new KeyValueTimestamp<>(-1, 2, 2), new KeyValueTimestamp<>(-1, 3, 3) };
    for (int i = 0; i < expected.length; i++) {
        assertEquals(expected[i], processor.theCapturedProcessor().processed().get(i));
    }
}

Example 45 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KTableAggregateTest method testRemoveOldBeforeAddNew.

@Test
public void testRemoveOldBeforeAddNew() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String input = "count-test-input";
    final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    builder.table(input, consumed).groupBy((key, value) -> KeyValue.pair(String.valueOf(key.charAt(0)), String.valueOf(key.charAt(1))), stringSerialized).aggregate(() -> "", (aggKey, value, aggregate) -> aggregate + value, (key, value, aggregate) -> aggregate.replaceAll(value, ""), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("someStore").withValueSerde(Serdes.String())).toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), CONFIG, Instant.ofEpochMilli(0L))) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final MockApiProcessor<String, String, Void, Void> proc = supplier.theCapturedProcessor();
        inputTopic.pipeInput("11", "A", 10L);
        inputTopic.pipeInput("12", "B", 8L);
        inputTopic.pipeInput("11", (String) null, 12L);
        inputTopic.pipeInput("12", "C", 6L);
        assertEquals(asList(new KeyValueTimestamp<>("1", "1", 10), new KeyValueTimestamp<>("1", "12", 10), new KeyValueTimestamp<>("1", "2", 12), new KeyValueTimestamp<>("1", "", 12), new KeyValueTimestamp<>("1", "2", 12L)), proc.processed());
    }
}