Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 21 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamWindowAggregateTest method testAggBasic.

@Test
public void testAggBasic() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final KTable<Windowed<String>, String> table2 = builder.stream(topic1, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(TimeWindows.ofSizeAndGrace(ofMillis(10), ofMillis(100)).advanceBy(ofMillis(5))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.<String, String, WindowStore<Bytes, byte[]>>as("topic1-Canonized").withValueSerde(Serdes.String()));
    final MockApiProcessorSupplier<Windowed<String>, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table2.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        inputTopic1.pipeInput("A", "1", 0L);
        inputTopic1.pipeInput("B", "2", 1L);
        inputTopic1.pipeInput("C", "3", 2L);
        inputTopic1.pipeInput("D", "4", 3L);
        inputTopic1.pipeInput("A", "1", 4L);
        inputTopic1.pipeInput("A", "1", 5L);
        inputTopic1.pipeInput("B", "2", 6L);
        inputTopic1.pipeInput("D", "4", 7L);
        inputTopic1.pipeInput("B", "2", 8L);
        inputTopic1.pipeInput("C", "3", 9L);
        inputTopic1.pipeInput("A", "1", 10L);
        inputTopic1.pipeInput("B", "2", 11L);
        inputTopic1.pipeInput("D", "4", 12L);
        inputTopic1.pipeInput("B", "2", 13L);
        inputTopic1.pipeInput("C", "3", 14L);
        inputTopic1.pipeInput("B", "1", 3L);
        inputTopic1.pipeInput("B", "2", 2L);
        inputTopic1.pipeInput("B", "3", 9L);
    }
    assertEquals(asList(new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1", 0), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2", 1), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3", 2), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4", 3), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1", 4), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1+1+1", 5), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1", 5), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2", 6), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2", 6), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(0, 10)), "0+4+4", 7), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+4", 7), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2", 8), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2", 8), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(0, 10)), "0+3+3", 9), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(5, 15)), "0+3", 9), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+1", 10), new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(10, 20)), "0+1", 10), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2+2", 11), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(10, 20)), "0+2", 11), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(5, 15)), "0+4+4", 12), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(10, 20)), "0+4", 12), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2+2+2", 13), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(10, 20)), "0+2+2", 13), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(5, 15)), "0+3+3", 14), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(10, 20)), "0+3", 14), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2+1", 8), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2+1+2", 8), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(0, 10)), "0+2+2+2+1+2+3", 9), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(5, 15)), "0+2+2+2+2+3", 13)), supplier.theCapturedProcessor().processed());
}

Example 22 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KTableAggregateTest method testAggRepartition.

@Test
public void testAggRepartition() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final KTable<String, String> table1 = builder.table(topic1, consumed);
    final KTable<String, String> table2 = table1.groupBy((key, value) -> {
        switch(key) {
            case "null":
                return KeyValue.pair(null, value);
            case "NULL":
                return null;
            default:
                return KeyValue.pair(value, value);
        }
    }, stringSerialized).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("topic1-Canonized").withValueSerde(stringSerde));
    table2.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), CONFIG, Instant.ofEpochMilli(0L))) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        inputTopic.pipeInput("A", "1", 10L);
        inputTopic.pipeInput("A", (String) null, 15L);
        inputTopic.pipeInput("A", "1", 12L);
        inputTopic.pipeInput("B", "2", 20L);
        inputTopic.pipeInput("null", "3", 25L);
        inputTopic.pipeInput("B", "4", 23L);
        inputTopic.pipeInput("NULL", "5", 24L);
        inputTopic.pipeInput("B", "7", 22L);
        assertEquals(asList(new KeyValueTimestamp<>("1", "0+1", 10), new KeyValueTimestamp<>("1", "0+1-1", 15), new KeyValueTimestamp<>("1", "0+1-1+1", 15), new KeyValueTimestamp<>("2", "0+2", 20), new KeyValueTimestamp<>("2", "0+2-2", 23), new KeyValueTimestamp<>("4", "0+4", 23), new KeyValueTimestamp<>("4", "0+4-4", 23), new KeyValueTimestamp<>("7", "0+7", 22)), supplier.theCapturedProcessor().processed());
    }
}

Example 23 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KTableAggregateTest method testCountHelper.

private static void testCountHelper(final StreamsBuilder builder, final String input, final MockApiProcessorSupplier<String, Object, Void, Void> supplier) {
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), CONFIG, Instant.ofEpochMilli(0L))) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        inputTopic.pipeInput("A", "green", 10L);
        inputTopic.pipeInput("B", "green", 9L);
        inputTopic.pipeInput("A", "blue", 12L);
        inputTopic.pipeInput("C", "yellow", 15L);
        inputTopic.pipeInput("D", "green", 11L);
        assertEquals(asList(new KeyValueTimestamp<>("green", 1L, 10), new KeyValueTimestamp<>("green", 2L, 10), new KeyValueTimestamp<>("green", 1L, 12), new KeyValueTimestamp<>("blue", 1L, 12), new KeyValueTimestamp<>("yellow", 1L, 15), new KeyValueTimestamp<>("green", 2L, 12)), supplier.theCapturedProcessor().processed());
    }
}

Example 24 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KTableImplTest method testMaterializedKTable.

@Test
public void testMaterializedKTable() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String topic2 = "topic2";
    final KTable<String, String> table1 = builder.table(topic1, consumed, Materialized.as("fred"));
    final MockApiProcessorSupplier<String, Object, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1.toStream().process(supplier);
    final KTable<String, Integer> table2 = table1.mapValues(s -> Integer.valueOf(s));
    table2.toStream().process(supplier);
    final KTable<String, Integer> table3 = table2.filter((key, value) -> (value % 2) == 0);
    table3.toStream().process(supplier);
    table1.toStream().to(topic2, produced);
    final KTable<String, String> table4 = builder.table(topic2, consumed);
    table4.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
        inputTopic.pipeInput("A", "01", 5L);
        inputTopic.pipeInput("B", "02", 100L);
        inputTopic.pipeInput("C", "03", 0L);
        inputTopic.pipeInput("D", "04", 0L);
        inputTopic.pipeInput("A", "05", 10L);
        inputTopic.pipeInput("A", "06", 8L);
    }
    final List<MockApiProcessor<String, Object, Void, Void>> processors = supplier.capturedProcessors(4);
    assertEquals(asList(new KeyValueTimestamp<>("A", "01", 5), new KeyValueTimestamp<>("B", "02", 100), new KeyValueTimestamp<>("C", "03", 0), new KeyValueTimestamp<>("D", "04", 0), new KeyValueTimestamp<>("A", "05", 10), new KeyValueTimestamp<>("A", "06", 8)), processors.get(0).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", 1, 5), new KeyValueTimestamp<>("B", 2, 100), new KeyValueTimestamp<>("C", 3, 0), new KeyValueTimestamp<>("D", 4, 0), new KeyValueTimestamp<>("A", 5, 10), new KeyValueTimestamp<>("A", 6, 8)), processors.get(1).processed());
    assertEquals(asList(new KeyValueTimestamp<>("B", 2, 100), new KeyValueTimestamp<>("D", 4, 0), new KeyValueTimestamp<>("A", 6, 8)), processors.get(2).processed());
    assertEquals(asList(new KeyValueTimestamp<>("A", "01", 5), new KeyValueTimestamp<>("B", "02", 100), new KeyValueTimestamp<>("C", "03", 0), new KeyValueTimestamp<>("D", "04", 0), new KeyValueTimestamp<>("A", "05", 10), new KeyValueTimestamp<>("A", "06", 8)), processors.get(3).processed());
}

Example 25 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KTableSourceTest method testKTable.

@Test
public void testKTable() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final KTable<String, Integer> table1 = builder.table(topic1, Consumed.with(Serdes.String(), Serdes.Integer()));
    final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1.toStream().process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, Integer> inputTopic = driver.createInputTopic(topic1, new StringSerializer(), new IntegerSerializer());
        inputTopic.pipeInput("A", 1, 10L);
        inputTopic.pipeInput("B", 2, 11L);
        inputTopic.pipeInput("C", 3, 12L);
        inputTopic.pipeInput("D", 4, 13L);
        inputTopic.pipeInput("A", null, 14L);
        inputTopic.pipeInput("B", null, 15L);
    }
    assertEquals(asList(new KeyValueTimestamp<>("A", 1, 10L), new KeyValueTimestamp<>("B", 2, 11L), new KeyValueTimestamp<>("C", 3, 12L), new KeyValueTimestamp<>("D", 4, 13L), new KeyValueTimestamp<>("A", null, 14L), new KeyValueTimestamp<>("B", null, 15L)), supplier.theCapturedProcessor().processed());
}