Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 66 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressScenarioTest method shouldWorkBeforeGroupBy.

@Test
public void shouldWorkBeforeGroupBy() {
    final StreamsBuilder builder = new StreamsBuilder();
    builder.table("topic", Consumed.with(Serdes.String(), Serdes.String())).suppress(untilTimeLimit(ofMillis(10), unbounded())).groupBy(KeyValue::pair, Grouped.with(Serdes.String(), Serdes.String())).count().toStream().to("output", Produced.with(Serdes.String(), Serdes.Long()));
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), config)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic("topic", STRING_SERIALIZER, STRING_SERIALIZER);
        inputTopic.pipeInput("A", "a", 0L);
        inputTopic.pipeInput("tick", "tick", 10L);
        verify(drainProducerRecords(driver, "output", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("A", 1L, 0L)));
    }
}

Example 67 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressScenarioTest method shouldSupportFinalResultsForTimeWindows.

@Test
public void shouldSupportFinalResultsForTimeWindows() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<Windowed<String>, Long> valueCounts = builder.stream("input", Consumed.with(STRING_SERDE, STRING_SERDE)).groupBy((String k, String v) -> k, Grouped.with(STRING_SERDE, STRING_SERDE)).windowedBy(TimeWindows.of(ofMillis(2L)).grace(ofMillis(1L))).count(Materialized.<String, Long, WindowStore<Bytes, byte[]>>as("counts").withCachingDisabled());
    valueCounts.suppress(untilWindowCloses(unbounded())).toStream().map((final Windowed<String> k, final Long v) -> new KeyValue<>(k.toString(), v)).to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().map((final Windowed<String> k, final Long v) -> new KeyValue<>(k.toString(), v)).to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
    final Topology topology = builder.build();
    System.out.println(topology.describe());
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
        inputTopic.pipeInput("k1", "v1", 0L);
        inputTopic.pipeInput("k1", "v1", 1L);
        inputTopic.pipeInput("k1", "v1", 2L);
        inputTopic.pipeInput("k1", "v1", 1L);
        inputTopic.pipeInput("k1", "v1", 0L);
        inputTopic.pipeInput("k1", "v1", 5L);
        // note this last record gets dropped because it is out of the grace period
        inputTopic.pipeInput("k1", "v1", 0L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("[k1@0/2]", 1L, 0L), new KeyValueTimestamp<>("[k1@0/2]", 2L, 1L), new KeyValueTimestamp<>("[k1@2/4]", 1L, 2L), new KeyValueTimestamp<>("[k1@0/2]", 3L, 1L), new KeyValueTimestamp<>("[k1@0/2]", 4L, 1L), new KeyValueTimestamp<>("[k1@4/6]", 1L, 5L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("[k1@0/2]", 4L, 1L), new KeyValueTimestamp<>("[k1@2/4]", 1L, 2L)));
    }
}

Example 68 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressScenarioTest method shouldSuppressIntermediateEventsWithTimeLimit.

@Test
public void shouldSuppressIntermediateEventsWithTimeLimit() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, Long> valueCounts = builder.table("input", Consumed.with(STRING_SERDE, STRING_SERDE), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>with(STRING_SERDE, STRING_SERDE).withCachingDisabled().withLoggingDisabled()).groupBy((k, v) -> new KeyValue<>(v, k), Grouped.with(STRING_SERDE, STRING_SERDE)).count();
    valueCounts.suppress(untilTimeLimit(ofMillis(2L), unbounded())).toStream().to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
    final Topology topology = builder.build();
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
        inputTopic.pipeInput("k1", "v1", 0L);
        inputTopic.pipeInput("k1", "v2", 1L);
        inputTopic.pipeInput("k2", "v1", 2L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("v1", 1L, 0L), new KeyValueTimestamp<>("v1", 0L, 1L), new KeyValueTimestamp<>("v2", 1L, 1L), new KeyValueTimestamp<>("v1", 1L, 2L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("v1", 1L, 2L)));
        // inserting a dummy "tick" record just to advance stream time
        inputTopic.pipeInput("tick", "tick", 3L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("tick", 1L, 3L)));
        // the stream time is now 3, so it's time to emit this record
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("v2", 1L, 1L)));
        inputTopic.pipeInput("tick", "tock", 4L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("tick", 0L, 4L), new KeyValueTimestamp<>("tock", 1L, 4L)));
        // tick is still buffered, since it was first inserted at time 3, and it is only time 4 right now.
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), emptyList());
    }
}

Example 69 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class EOSUncleanShutdownIntegrationTest method shouldWorkWithUncleanShutdownWipeOutStateStore.

@Test
public void shouldWorkWithUncleanShutdownWipeOutStateStore() throws InterruptedException {
    final String appId = "shouldWorkWithUncleanShutdownWipeOutStateStore";
    STREAMS_CONFIG.put(StreamsConfig.APPLICATION_ID_CONFIG, appId);
    STREAMS_CONFIG.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, eosConfig);
    final String input = "input-topic";
    cleanStateBeforeTest(CLUSTER, input);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    final AtomicInteger recordCount = new AtomicInteger(0);
    final KTable<String, String> valueCounts = inputStream.groupByKey().aggregate(() -> "()", (key, value, aggregate) -> aggregate + ",(" + key + ": " + value + ")", Materialized.as("aggregated_value"));
    valueCounts.toStream().peek((key, value) -> {
        if (recordCount.incrementAndGet() >= RECORD_TOTAL) {
            throw new IllegalStateException("Crash on the " + RECORD_TOTAL + " record");
        }
    });
    final Properties producerConfig = mkProperties(mkMap(mkEntry(ProducerConfig.CLIENT_ID_CONFIG, "anything"), mkEntry(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ((Serializer<String>) STRING_SERIALIZER).getClass().getName()), mkEntry(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ((Serializer<String>) STRING_SERIALIZER).getClass().getName()), mkEntry(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers())));
    final KafkaStreams driver = new KafkaStreams(builder.build(), STREAMS_CONFIG);
    driver.cleanUp();
    driver.start();
    // Task's StateDir
    final File taskStateDir = new File(String.join("/", TEST_FOLDER.getRoot().getPath(), appId, "0_0"));
    final File taskCheckpointFile = new File(taskStateDir, ".checkpoint");
    try {
        IntegrationTestUtils.produceSynchronously(producerConfig, false, input, Optional.empty(), singletonList(new KeyValueTimestamp<>("k1", "v1", 0L)));
        // wait until the first request is processed and some files are created in it
        TestUtils.waitForCondition(() -> taskStateDir.exists() && taskStateDir.isDirectory() && taskStateDir.list().length > 0, "Failed awaiting CreateTopics first request failure");
        IntegrationTestUtils.produceSynchronously(producerConfig, false, input, Optional.empty(), asList(new KeyValueTimestamp<>("k2", "v2", 1L), new KeyValueTimestamp<>("k3", "v3", 2L)));
        TestUtils.waitForCondition(() -> recordCount.get() == RECORD_TOTAL, "Expected " + RECORD_TOTAL + " records processed but only got " + recordCount.get());
    } finally {
        TestUtils.waitForCondition(() -> driver.state().equals(State.ERROR), "Expected ERROR state but driver is on " + driver.state());
        driver.close();
        // Although there is an uncaught exception,
        // case 1: the state directory is cleaned up without any problems.
        // case 2: The state directory is not cleaned up, for it does not include any checkpoint file.
        // case 3: The state directory is not cleaned up, for it includes a checkpoint file but it is empty.
        assertTrue(!taskStateDir.exists() || (taskStateDir.exists() && taskStateDir.list().length > 0 && !taskCheckpointFile.exists()) || (taskCheckpointFile.exists() && taskCheckpointFile.length() == 0L));
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}

Example 70 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class CachingInMemorySessionStoreTest method shouldNotForwardChangedValuesDuringFlushWhenSendOldValuesDisabled.

@Test
public void shouldNotForwardChangedValuesDuringFlushWhenSendOldValuesDisabled() {
    final Windowed<Bytes> a = new Windowed<>(keyA, new SessionWindow(0, 0));
    final Windowed<String> aDeserialized = new Windowed<>("a", new SessionWindow(0, 0));
    final CacheFlushListenerStub<Windowed<String>, String> flushListener = new CacheFlushListenerStub<>(new SessionWindowedDeserializer<>(new StringDeserializer()), new StringDeserializer());
    cachingStore.setFlushListener(flushListener, false);
    cachingStore.put(a, "1".getBytes());
    cachingStore.flush();
    cachingStore.put(a, "2".getBytes());
    cachingStore.flush();
    cachingStore.remove(a);
    cachingStore.flush();
    assertEquals(asList(new KeyValueTimestamp<>(aDeserialized, new Change<>("1", null), DEFAULT_TIMESTAMP), new KeyValueTimestamp<>(aDeserialized, new Change<>("2", null), DEFAULT_TIMESTAMP), new KeyValueTimestamp<>(aDeserialized, new Change<>(null, null), DEFAULT_TIMESTAMP)), flushListener.forwarded);
    flushListener.forwarded.clear();
    cachingStore.put(a, "1".getBytes());
    cachingStore.put(a, "2".getBytes());
    cachingStore.remove(a);
    cachingStore.flush();
    assertEquals(Collections.emptyList(), flushListener.forwarded);
    flushListener.forwarded.clear();
}