Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 31 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KTableTransformValuesTest method shouldCalculateCorrectOldValuesIfMaterializedEvenIfStateful.

@Test
public void shouldCalculateCorrectOldValuesIfMaterializedEvenIfStateful() {
    builder.table(INPUT_TOPIC, CONSUMED).transformValues(new StatefulTransformerSupplier(), Materialized.<String, Integer, KeyValueStore<Bytes, byte[]>>as(QUERYABLE_NAME).withKeySerde(Serdes.String()).withValueSerde(Serdes.Integer())).groupBy(toForceSendingOfOldValues(), Grouped.with(Serdes.String(), Serdes.Integer())).reduce(MockReducer.INTEGER_ADDER, MockReducer.INTEGER_SUBTRACTOR).mapValues(mapBackToStrings()).toStream().process(capture);
    driver = new TopologyTestDriver(builder.build(), props());
    final TestInputTopic<String, String> inputTopic = driver.createInputTopic(INPUT_TOPIC, new StringSerializer(), new StringSerializer());
    inputTopic.pipeInput("A", "ignored", 5L);
    inputTopic.pipeInput("A", "ignored1", 15L);
    inputTopic.pipeInput("A", "ignored2", 10L);
    assertThat(output(), hasItems(new KeyValueTimestamp<>("A", "1", 5), new KeyValueTimestamp<>("A", "0", 15), new KeyValueTimestamp<>("A", "2", 15), new KeyValueTimestamp<>("A", "0", 15), new KeyValueTimestamp<>("A", "3", 15)));
    final KeyValueStore<String, Integer> keyValueStore = driver.getKeyValueStore(QUERYABLE_NAME);
    assertThat(keyValueStore.get("A"), is(3));
}

Example 32 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class CachingPersistentSessionStoreTest method shouldForwardChangedValuesDuringFlush.

@Test
public void shouldForwardChangedValuesDuringFlush() {
    final Windowed<Bytes> a = new Windowed<>(keyA, new SessionWindow(2, 4));
    final Windowed<Bytes> b = new Windowed<>(keyA, new SessionWindow(1, 2));
    final Windowed<String> aDeserialized = new Windowed<>("a", new SessionWindow(2, 4));
    final Windowed<String> bDeserialized = new Windowed<>("a", new SessionWindow(1, 2));
    final CacheFlushListenerStub<Windowed<String>, String> flushListener = new CacheFlushListenerStub<>(new SessionWindowedDeserializer<>(new StringDeserializer()), new StringDeserializer());
    cachingStore.setFlushListener(flushListener, true);
    cachingStore.put(b, "1".getBytes());
    cachingStore.flush();
    assertEquals(Collections.singletonList(new KeyValueTimestamp<>(bDeserialized, new Change<>("1", null), DEFAULT_TIMESTAMP)), flushListener.forwarded);
    flushListener.forwarded.clear();
    cachingStore.put(a, "1".getBytes());
    cachingStore.flush();
    assertEquals(Collections.singletonList(new KeyValueTimestamp<>(aDeserialized, new Change<>("1", null), DEFAULT_TIMESTAMP)), flushListener.forwarded);
    flushListener.forwarded.clear();
    cachingStore.put(a, "2".getBytes());
    cachingStore.flush();
    assertEquals(Collections.singletonList(new KeyValueTimestamp<>(aDeserialized, new Change<>("2", "1"), DEFAULT_TIMESTAMP)), flushListener.forwarded);
    flushListener.forwarded.clear();
    cachingStore.remove(a);
    cachingStore.flush();
    assertEquals(Collections.singletonList(new KeyValueTimestamp<>(aDeserialized, new Change<>(null, "2"), DEFAULT_TIMESTAMP)), flushListener.forwarded);
    flushListener.forwarded.clear();
    cachingStore.put(a, "1".getBytes());
    cachingStore.put(a, "2".getBytes());
    cachingStore.remove(a);
    cachingStore.flush();
    assertEquals(Collections.emptyList(), flushListener.forwarded);
    flushListener.forwarded.clear();
}

Example 33 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class CachingPersistentSessionStoreTest method shouldNotForwardChangedValuesDuringFlushWhenSendOldValuesDisabled.

@Test
public void shouldNotForwardChangedValuesDuringFlushWhenSendOldValuesDisabled() {
    final Windowed<Bytes> a = new Windowed<>(keyA, new SessionWindow(0, 0));
    final Windowed<String> aDeserialized = new Windowed<>("a", new SessionWindow(0, 0));
    final CacheFlushListenerStub<Windowed<String>, String> flushListener = new CacheFlushListenerStub<>(new SessionWindowedDeserializer<>(new StringDeserializer()), new StringDeserializer());
    cachingStore.setFlushListener(flushListener, false);
    cachingStore.put(a, "1".getBytes());
    cachingStore.flush();
    cachingStore.put(a, "2".getBytes());
    cachingStore.flush();
    cachingStore.remove(a);
    cachingStore.flush();
    assertEquals(asList(new KeyValueTimestamp<>(aDeserialized, new Change<>("1", null), DEFAULT_TIMESTAMP), new KeyValueTimestamp<>(aDeserialized, new Change<>("2", null), DEFAULT_TIMESTAMP), new KeyValueTimestamp<>(aDeserialized, new Change<>(null, null), DEFAULT_TIMESTAMP)), flushListener.forwarded);
    flushListener.forwarded.clear();
    cachingStore.put(a, "1".getBytes());
    cachingStore.put(a, "2".getBytes());
    cachingStore.remove(a);
    cachingStore.flush();
    assertEquals(Collections.emptyList(), flushListener.forwarded);
    flushListener.forwarded.clear();
}

Example 34 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldGroupByKey.

@SuppressWarnings("deprecation")
@Test
public void shouldGroupByKey() throws Exception {
    final long timestamp = mockTime.milliseconds();
    produceMessages(timestamp);
    produceMessages(timestamp);
    // noinspection deprecation
    stream.groupByKey(Grouped.with(Serdes.Integer(), Serdes.String())).windowedBy(TimeWindows.of(ofMillis(500L))).count().toStream((windowedKey, value) -> windowedKey.key() + "@" + windowedKey.window().start()).to(outputTopic, Produced.with(Serdes.String(), Serdes.Long()));
    startStreams();
    final List<KeyValueTimestamp<String, Long>> results = receiveMessages(new StringDeserializer(), new LongDeserializer(), 10);
    results.sort(KStreamAggregationIntegrationTest::compare);
    final long window = timestamp / 500 * 500;
    assertThat(results, is(Arrays.asList(new KeyValueTimestamp("1@" + window, 1L, timestamp), new KeyValueTimestamp("1@" + window, 2L, timestamp), new KeyValueTimestamp("2@" + window, 1L, timestamp), new KeyValueTimestamp("2@" + window, 2L, timestamp), new KeyValueTimestamp("3@" + window, 1L, timestamp), new KeyValueTimestamp("3@" + window, 2L, timestamp), new KeyValueTimestamp("4@" + window, 1L, timestamp), new KeyValueTimestamp("4@" + window, 2L, timestamp), new KeyValueTimestamp("5@" + window, 1L, timestamp), new KeyValueTimestamp("5@" + window, 2L, timestamp))));
}

Example 35 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldAggregateSlidingWindows.

@SuppressWarnings("deprecation")
@Test
public void shouldAggregateSlidingWindows() throws Exception {
    final long firstBatchTimestamp = mockTime.milliseconds();
    final long timeDifference = 500L;
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = firstBatchTimestamp + timeDifference / 2;
    produceMessages(secondBatchTimestamp);
    final long thirdBatchTimestamp = firstBatchTimestamp + timeDifference - 100L;
    produceMessages(thirdBatchTimestamp);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, timeDifference);
    // noinspection deprecation
    groupedStream.windowedBy(SlidingWindows.withTimeDifferenceAndGrace(ofMillis(500L), ofMinutes(5))).aggregate(initializer, aggregator, Materialized.with(null, Serdes.Integer())).toStream().to(outputTopic, Produced.with(windowedSerde, Serdes.Integer()));
    startStreams();
    final List<KeyValueTimestamp<Windowed<String>, Integer>> windowedMessages = receiveMessagesWithTimestamp(new TimeWindowedDeserializer<>(), new IntegerDeserializer(), String.class, 30);
    // read from ConsoleConsumer
    final String resultFromConsoleConsumer = readWindowedKeyedMessagesViaConsoleConsumer(new TimeWindowedDeserializer<String>(), new IntegerDeserializer(), String.class, 30, true);
    final Comparator<KeyValueTimestamp<Windowed<String>, Integer>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, Integer> o) -> o.key().key()).thenComparingInt(KeyValueTimestamp::value);
    windowedMessages.sort(comparator);
    final long firstBatchLeftWindowStart = firstBatchTimestamp - timeDifference;
    final long firstBatchLeftWindowEnd = firstBatchLeftWindowStart + timeDifference;
    final long firstBatchRightWindowStart = firstBatchTimestamp + 1;
    final long firstBatchRightWindowEnd = firstBatchRightWindowStart + timeDifference;
    final long secondBatchLeftWindowStart = secondBatchTimestamp - timeDifference;
    final long secondBatchLeftWindowEnd = secondBatchLeftWindowStart + timeDifference;
    final long secondBatchRightWindowStart = secondBatchTimestamp + 1;
    final long secondBatchRightWindowEnd = secondBatchRightWindowStart + timeDifference;
    final long thirdBatchLeftWindowStart = thirdBatchTimestamp - timeDifference;
    final long thirdBatchLeftWindowEnd = thirdBatchLeftWindowStart + timeDifference;
    final List<KeyValueTimestamp<Windowed<String>, Integer>> expectResult = Arrays.asList(// A @ firstBatchTimestamp left window created when A @ firstBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), // A @ firstBatchTimestamp right window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), // A @ secondBatchTimestamp right window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), // A @ secondBatchTimestamp left window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), // A @ firstBatchTimestamp right window updated when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), // A @ thirdBatchTimestamp left window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), 1, firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 1, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), 1, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), 2, secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), 2, thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), 3, thirdBatchTimestamp));
    assertThat(windowedMessages, is(expectResult));
    final Set<String> expectResultString = new HashSet<>(expectResult.size());
    for (final KeyValueTimestamp<Windowed<String>, Integer> eachRecord : expectResult) {
        expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value());
    }
    // check every message is contained in the expect result
    final String[] allRecords = resultFromConsoleConsumer.split("\n");
    for (final String record : allRecords) {
        assertTrue(expectResultString.contains(record));
    }
}