use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.
the class KStreamAggregationIntegrationTest method shouldReduce.
@Test
public void shouldReduce() throws Exception {
produceMessages(mockTime.milliseconds());
groupedStream.reduce(reducer, Materialized.as("reduce-by-key")).toStream().to(outputTopic, Produced.with(Serdes.String(), Serdes.String()));
startStreams();
produceMessages(mockTime.milliseconds());
final List<KeyValueTimestamp<String, String>> results = receiveMessages(new StringDeserializer(), new StringDeserializer(), 10);
results.sort(KStreamAggregationIntegrationTest::compare);
assertThat(results, is(Arrays.asList(new KeyValueTimestamp("A", "A", mockTime.milliseconds()), new KeyValueTimestamp("A", "A:A", mockTime.milliseconds()), new KeyValueTimestamp("B", "B", mockTime.milliseconds()), new KeyValueTimestamp("B", "B:B", mockTime.milliseconds()), new KeyValueTimestamp("C", "C", mockTime.milliseconds()), new KeyValueTimestamp("C", "C:C", mockTime.milliseconds()), new KeyValueTimestamp("D", "D", mockTime.milliseconds()), new KeyValueTimestamp("D", "D:D", mockTime.milliseconds()), new KeyValueTimestamp("E", "E", mockTime.milliseconds()), new KeyValueTimestamp("E", "E:E", mockTime.milliseconds()))));
}
use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.
the class KStreamSessionWindowAggregateProcessorTest method setup.
private void setup(final boolean enableCache) {
final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, "test", StreamsConfig.METRICS_LATEST, new MockTime());
context = new InternalMockProcessorContext<Windowed<String>, Change<Long>>(TestUtils.tempDirectory(), Serdes.String(), Serdes.String(), streamsMetrics, new StreamsConfig(StreamsTestUtils.getStreamsConfig()), MockRecordCollector::new, new ThreadCache(new LogContext("testCache "), 100000, streamsMetrics), Time.SYSTEM) {
@Override
public <K extends Windowed<String>, V extends Change<Long>> void forward(final Record<K, V> record) {
results.add(new KeyValueTimestamp<>(record.key(), record.value(), record.timestamp()));
}
};
// Set initial timestamp for CachingSessionStore to prepare entry from as default
// InternalMockProcessorContext#timestamp returns -1.
context.setTime(0L);
TaskMetrics.droppedRecordsSensor(threadId, context.taskId().toString(), streamsMetrics);
initStore(enableCache);
processor.init(context);
}
use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.
the class KStreamSlidingWindowAggregateTest method testAggregateLargeInput.
@Test
public void testAggregateLargeInput() {
final StreamsBuilder builder = new StreamsBuilder();
final String topic1 = "topic1";
final WindowBytesStoreSupplier storeSupplier = inOrderIterator ? new InOrderMemoryWindowStoreSupplier("InOrder", 50000L, 10L, false) : Stores.inMemoryWindowStore("Reverse", Duration.ofMillis(50000), Duration.ofMillis(10), false);
final KTable<Windowed<String>, String> table2 = builder.stream(topic1, Consumed.with(Serdes.String(), Serdes.String())).groupByKey(Grouped.with(Serdes.String(), Serdes.String())).windowedBy(SlidingWindows.ofTimeDifferenceAndGrace(ofMillis(10), ofMillis(50))).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, Materialized.as(storeSupplier));
final MockApiProcessorSupplier<Windowed<String>, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
table2.toStream().process(supplier);
try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer());
inputTopic1.pipeInput("A", "1", 10L);
inputTopic1.pipeInput("A", "2", 20L);
inputTopic1.pipeInput("A", "3", 22L);
inputTopic1.pipeInput("A", "4", 15L);
inputTopic1.pipeInput("B", "1", 12L);
inputTopic1.pipeInput("B", "2", 13L);
inputTopic1.pipeInput("B", "3", 18L);
inputTopic1.pipeInput("B", "4", 19L);
inputTopic1.pipeInput("B", "5", 25L);
inputTopic1.pipeInput("B", "6", 14L);
inputTopic1.pipeInput("C", "1", 11L);
inputTopic1.pipeInput("C", "2", 15L);
inputTopic1.pipeInput("C", "3", 16L);
inputTopic1.pipeInput("C", "4", 21);
inputTopic1.pipeInput("C", "5", 23L);
inputTopic1.pipeInput("D", "4", 11L);
inputTopic1.pipeInput("D", "2", 12L);
inputTopic1.pipeInput("D", "3", 29L);
inputTopic1.pipeInput("D", "5", 16L);
}
final Comparator<KeyValueTimestamp<Windowed<String>, String>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, String> o) -> o.key().key()).thenComparing((KeyValueTimestamp<Windowed<String>, String> o) -> o.key().window().start());
final ArrayList<KeyValueTimestamp<Windowed<String>, String>> actual = supplier.theCapturedProcessor().processed();
actual.sort(comparator);
assertEquals(asList(// FINAL WINDOW: A@10 left window created when A@10 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(0, 10)), "0+1", 10), // FINAL WINDOW: A@15 left window created when A@15 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(5, 15)), "0+1+4", 15), // A@20 left window created when A@20 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(10, 20)), "0+1+2", 20), // FINAL WINDOW: A@20 left window updated when A@15 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(10, 20)), "0+1+2+4", 20), // A@10 right window created when A@20 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(11, 21)), "0+2", 20), // FINAL WINDOW: A@10 right window updated when A@15 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(11, 21)), "0+2+4", 20), // A@22 left window created when A@22 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(12, 22)), "0+2+3", 22), // FINAL WINDOW: A@22 left window updated when A@15 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(12, 22)), "0+2+3+4", 22), // FINAL WINDOW: A@15 right window created when A@15 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(16, 26)), "0+2+3", 22), // FINAL WINDOW: A@20 right window created when A@22 processed
new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(21, 31)), "0+3", 22), // FINAL WINDOW: B@12 left window created when B@12 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(2, 12)), "0+1", 12), // FINAL WINDOW: B@13 left window created when B@13 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(3, 13)), "0+1+2", 13), // FINAL WINDOW: B@14 left window created when B@14 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(4, 14)), "0+1+2+6", 14), // B@18 left window created when B@18 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(8, 18)), "0+1+2+3", 18), // FINAL WINDOW: B@18 left window updated when B@14 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(8, 18)), "0+1+2+3+6", 18), // B@19 left window created when B@19 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(9, 19)), "0+1+2+3+4", 19), // FINAL WINDOW: B@19 left window updated when B@14 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(9, 19)), "0+1+2+3+4+6", 19), // B@12 right window created when B@13 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2", 13), // B@12 right window updated when B@18 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2+3", 18), // B@12 right window updated when B@19 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2+3+4", 19), // FINAL WINDOW: B@12 right window updated when B@14 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(13, 23)), "0+2+3+4+6", 19), // B@13 right window created when B@18 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(14, 24)), "0+3", 18), // B@13 right window updated when B@19 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(14, 24)), "0+3+4", 19), // FINAL WINDOW: B@13 right window updated when B@14 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(14, 24)), "0+3+4+6", 19), // FINAL WINDOW: B@25 left window created when B@25 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(15, 25)), "0+3+4+5", 25), // B@18 right window created when B@19 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(19, 29)), "0+4", 19), // FINAL WINDOW: B@18 right window updated when B@25 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(19, 29)), "0+4+5", 25), // FINAL WINDOW: B@19 right window updated when B@25 processed
new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(20, 30)), "0+5", 25), // FINAL WINDOW: C@11 left window created when C@11 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(1, 11)), "0+1", 11), // FINAL WINDOW: C@15 left window created when C@15 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(5, 15)), "0+1+2", 15), // FINAL WINDOW: C@16 left window created when C@16 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(6, 16)), "0+1+2+3", 16), // FINAL WINDOW: C@21 left window created when C@21 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(11, 21)), "0+1+2+3+4", 21), // C@11 right window created when C@15 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(12, 22)), "0+2", 15), // C@11 right window updated when C@16 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(12, 22)), "0+2+3", 16), // FINAL WINDOW: C@11 right window updated when C@21 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(12, 22)), "0+2+3+4", 21), // FINAL WINDOW: C@23 left window created when C@23 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(13, 23)), "0+2+3+4+5", 23), // C@15 right window created when C@16 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(16, 26)), "0+3", 16), // C@15 right window updated when C@21 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(16, 26)), "0+3+4", 21), // FINAL WINDOW: C@15 right window updated when C@23 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(16, 26)), "0+3+4+5", 23), // C@16 right window created when C@21 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(17, 27)), "0+4", 21), // FINAL WINDOW: C@16 right window updated when C@23 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(17, 27)), "0+4+5", 23), // FINAL WINDOW: C@21 right window created when C@23 processed
new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(22, 32)), "0+5", 23), // FINAL WINDOW: D@11 left window created when D@11 processed
new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(1, 11)), "0+4", 11), // FINAL WINDOW: D@12 left window created when D@12 processed
new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(2, 12)), "0+4+2", 12), // FINAL WINDOW: D@16 left window created when D@16 processed
new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(6, 16)), "0+4+2+5", 16), // D@11 right window created when D@12 processed
new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(12, 22)), "0+2", 12), // FINAL WINDOW: D@11 right window updated when D@16 processed
new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(12, 22)), "0+2+5", 16), // FINAL WINDOW: D@12 right window created when D@16 processed
new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(13, 23)), "0+5", 16), // FINAL WINDOW: D@29 left window created when D@29 processed
new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(19, 29)), "0+3", 29)), actual);
}
use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.
the class KStreamTransformTest method testTransformWithNewDriverAndPunctuator.
// Old PAPI. Needs to be migrated.
@SuppressWarnings("deprecation")
@Test
public void testTransformWithNewDriverAndPunctuator() {
final StreamsBuilder builder = new StreamsBuilder();
final TransformerSupplier<Number, Number, KeyValue<Integer, Integer>> transformerSupplier = () -> new Transformer<Number, Number, KeyValue<Integer, Integer>>() {
private int total = 0;
@Override
public void init(final ProcessorContext context) {
context.schedule(Duration.ofMillis(1), PunctuationType.WALL_CLOCK_TIME, timestamp -> context.forward(-1, (int) timestamp, To.all().withTimestamp(timestamp)));
}
@Override
public KeyValue<Integer, Integer> transform(final Number key, final Number value) {
total += value.intValue();
return KeyValue.pair(key.intValue() * 2, total);
}
@Override
public void close() {
}
};
final int[] expectedKeys = { 1, 10, 100, 1000 };
final MockProcessorSupplier<Integer, Integer> processor = new MockProcessorSupplier<>();
final KStream<Integer, Integer> stream = builder.stream(TOPIC_NAME, Consumed.with(Serdes.Integer(), Serdes.Integer()));
stream.transform(transformerSupplier).process(processor);
try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props, Instant.ofEpochMilli(0L))) {
final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(TOPIC_NAME, new IntegerSerializer(), new IntegerSerializer());
for (final int expectedKey : expectedKeys) {
inputTopic.pipeInput(expectedKey, expectedKey * 10, 0L);
}
// This tick yields the "-1:2" result
driver.advanceWallClockTime(Duration.ofMillis(2));
// This tick further advances the clock to 3, which leads to the "-1:3" result
driver.advanceWallClockTime(Duration.ofMillis(1));
}
assertEquals(6, processor.theCapturedProcessor().processed().size());
final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(2, 10, 0), new KeyValueTimestamp<>(20, 110, 0), new KeyValueTimestamp<>(200, 1110, 0), new KeyValueTimestamp<>(2000, 11110, 0), new KeyValueTimestamp<>(-1, 2, 2), new KeyValueTimestamp<>(-1, 3, 3) };
for (int i = 0; i < expected.length; i++) {
assertEquals(expected[i], processor.theCapturedProcessor().processed().get(i));
}
}
use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.
the class KTableAggregateTest method testRemoveOldBeforeAddNew.
@Test
public void testRemoveOldBeforeAddNew() {
final StreamsBuilder builder = new StreamsBuilder();
final String input = "count-test-input";
final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
builder.table(input, consumed).groupBy((key, value) -> KeyValue.pair(String.valueOf(key.charAt(0)), String.valueOf(key.charAt(1))), stringSerialized).aggregate(() -> "", (aggKey, value, aggregate) -> aggregate + value, (key, value, aggregate) -> aggregate.replaceAll(value, ""), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("someStore").withValueSerde(Serdes.String())).toStream().process(supplier);
try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), CONFIG, Instant.ofEpochMilli(0L))) {
final TestInputTopic<String, String> inputTopic = driver.createInputTopic(input, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
final MockApiProcessor<String, String, Void, Void> proc = supplier.theCapturedProcessor();
inputTopic.pipeInput("11", "A", 10L);
inputTopic.pipeInput("12", "B", 8L);
inputTopic.pipeInput("11", (String) null, 12L);
inputTopic.pipeInput("12", "C", 6L);
assertEquals(asList(new KeyValueTimestamp<>("1", "1", 10), new KeyValueTimestamp<>("1", "12", 10), new KeyValueTimestamp<>("1", "2", 12), new KeyValueTimestamp<>("1", "", 12), new KeyValueTimestamp<>("1", "2", 12L)), proc.processed());
}
}
Aggregations