Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 11 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class ResetPartitionTimeIntegrationTest method shouldPreservePartitionTimeOnKafkaStreamRestart.

@Test
public void shouldPreservePartitionTimeOnKafkaStreamRestart() {
    final String appId = "app-" + safeUniqueTestName(getClass(), testName);
    final String input = "input";
    final String outputRaw = "output-raw";
    cleanStateBeforeTest(CLUSTER, 2, input, outputRaw);
    final StreamsBuilder builder = new StreamsBuilder();
    builder.stream(input, Consumed.with(STRING_SERDE, STRING_SERDE)).to(outputRaw);
    final Properties streamsConfig = new Properties();
    streamsConfig.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, MaxTimestampExtractor.class);
    streamsConfig.put(StreamsConfig.APPLICATION_ID_CONFIG, appId);
    streamsConfig.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers());
    streamsConfig.put(StreamsConfig.POLL_MS_CONFIG, Integer.toString(DEFAULT_TIMEOUT));
    streamsConfig.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, (long) DEFAULT_TIMEOUT);
    streamsConfig.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, processingGuarantee);
    streamsConfig.put(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath());
    KafkaStreams kafkaStreams = getStartedStreams(streamsConfig, builder, true);
    try {
        // start sending some records to have partition time committed
        produceSynchronouslyToPartitionZero(input, Collections.singletonList(new KeyValueTimestamp<>("k3", "v3", 5000)));
        verifyOutput(outputRaw, Collections.singletonList(new KeyValueTimestamp<>("k3", "v3", 5000)));
        assertThat(lastRecordedTimestamp, is(-1L));
        lastRecordedTimestamp = -2L;
        kafkaStreams.close();
        assertThat(kafkaStreams.state(), is(KafkaStreams.State.NOT_RUNNING));
        kafkaStreams = getStartedStreams(streamsConfig, builder, true);
        // resend some records and retrieve the last committed timestamp
        produceSynchronouslyToPartitionZero(input, Collections.singletonList(new KeyValueTimestamp<>("k5", "v5", 4999)));
        verifyOutput(outputRaw, Collections.singletonList(new KeyValueTimestamp<>("k5", "v5", 4999)));
        assertThat(lastRecordedTimestamp, is(5000L));
    } finally {
        kafkaStreams.close();
        quietlyCleanStateAfterTest(CLUSTER, kafkaStreams);
    }
}

Example 12 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamAggregationIntegrationTest method shouldReduceSlidingWindows.

@SuppressWarnings("deprecation")
@Test
public void shouldReduceSlidingWindows() throws Exception {
    final long firstBatchTimestamp = mockTime.milliseconds();
    final long timeDifference = 500L;
    produceMessages(firstBatchTimestamp);
    final long secondBatchTimestamp = firstBatchTimestamp + timeDifference / 2;
    produceMessages(secondBatchTimestamp);
    final long thirdBatchTimestamp = firstBatchTimestamp + timeDifference - 100L;
    produceMessages(thirdBatchTimestamp);
    final Serde<Windowed<String>> windowedSerde = WindowedSerdes.timeWindowedSerdeFrom(String.class, timeDifference);
    // noinspection deprecation
    groupedStream.windowedBy(SlidingWindows.withTimeDifferenceAndGrace(ofMillis(timeDifference), ofMillis(2000L))).reduce(reducer).toStream().to(outputTopic, Produced.with(windowedSerde, Serdes.String()));
    startStreams();
    final List<KeyValueTimestamp<Windowed<String>, String>> windowedOutput = receiveMessages(new TimeWindowedDeserializer<>(new StringDeserializer(), 500L), new StringDeserializer(), String.class, 30);
    final String resultFromConsoleConsumer = readWindowedKeyedMessagesViaConsoleConsumer(new TimeWindowedDeserializer<String>(), new StringDeserializer(), String.class, 30, true);
    final Comparator<KeyValueTimestamp<Windowed<String>, String>> comparator = Comparator.comparing((KeyValueTimestamp<Windowed<String>, String> o) -> o.key().key()).thenComparing(KeyValueTimestamp::value);
    windowedOutput.sort(comparator);
    final long firstBatchLeftWindowStart = firstBatchTimestamp - timeDifference;
    final long firstBatchLeftWindowEnd = firstBatchLeftWindowStart + timeDifference;
    final long firstBatchRightWindowStart = firstBatchTimestamp + 1;
    final long firstBatchRightWindowEnd = firstBatchRightWindowStart + timeDifference;
    final long secondBatchLeftWindowStart = secondBatchTimestamp - timeDifference;
    final long secondBatchLeftWindowEnd = secondBatchLeftWindowStart + timeDifference;
    final long secondBatchRightWindowStart = secondBatchTimestamp + 1;
    final long secondBatchRightWindowEnd = secondBatchRightWindowStart + timeDifference;
    final long thirdBatchLeftWindowStart = thirdBatchTimestamp - timeDifference;
    final long thirdBatchLeftWindowEnd = thirdBatchLeftWindowStart + timeDifference;
    final List<KeyValueTimestamp<Windowed<String>, String>> expectResult = Arrays.asList(// A @ firstBatchTimestamp left window created when A @ firstBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "A", firstBatchTimestamp), // A @ firstBatchTimestamp right window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "A", secondBatchTimestamp), // A @ secondBatchTimestamp right window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "A", thirdBatchTimestamp), // A @ secondBatchTimestamp left window created when A @ secondBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "A:A", secondBatchTimestamp), // A @ firstBatchTimestamp right window updated when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "A:A", thirdBatchTimestamp), // A @ thirdBatchTimestamp left window created when A @ thirdBatchTimestamp processed
    new KeyValueTimestamp<>(new Windowed<>("A", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "A:A:A", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "B", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "B", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "B", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "B:B", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "B:B", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("B", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "B:B:B", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "C", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "C", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "C", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "C:C", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "C:C", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("C", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "C:C:C", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "D", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "D", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "D", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "D:D", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "D:D", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("D", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "D:D:D", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchLeftWindowStart, firstBatchLeftWindowEnd)), "E", firstBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "E", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchRightWindowStart, secondBatchRightWindowEnd)), "E", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(secondBatchLeftWindowStart, secondBatchLeftWindowEnd)), "E:E", secondBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(firstBatchRightWindowStart, firstBatchRightWindowEnd)), "E:E", thirdBatchTimestamp), new KeyValueTimestamp<>(new Windowed<>("E", new TimeWindow(thirdBatchLeftWindowStart, thirdBatchLeftWindowEnd)), "E:E:E", thirdBatchTimestamp));
    assertThat(windowedOutput, is(expectResult));
    final Set<String> expectResultString = new HashSet<>(expectResult.size());
    for (final KeyValueTimestamp<Windowed<String>, String> eachRecord : expectResult) {
        expectResultString.add("CreateTime:" + eachRecord.timestamp() + ", " + eachRecord.key() + ", " + eachRecord.value());
    }
    // check every message is contained in the expect result
    final String[] allRecords = resultFromConsoleConsumer.split("\n");
    for (final String record : allRecords) {
        assertTrue(expectResultString.contains(record));
    }
}

Example 13 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressionIntegrationTest method shouldInheritSerdes.

@Test
public void shouldInheritSerdes() {
    final String testId = "-shouldInheritSerdes";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    // count sets the serde to Long
    final KTable<String, Long> valueCounts = inputStream.groupByKey().count();
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).emitEarlyWhenFull())).toStream().to(outputSuppressed);
    valueCounts.toStream().to(outputRaw);
    final Properties streamsConfig = getStreamsConfig(appId);
    streamsConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    streamsConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        final boolean rawRecords = waitForAnyRecord(outputRaw);
        final boolean suppressedRecords = waitForAnyRecord(outputSuppressed);
        assertThat(rawRecords, Matchers.is(true));
        assertThat(suppressedRecords, is(true));
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}

Example 14 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressionIntegrationTest method shouldShutdownWhenRecordConstraintIsViolated.

@Test
public void shouldShutdownWhenRecordConstraintIsViolated() throws InterruptedException {
    final String testId = "-shouldShutdownWhenRecordConstraintIsViolated";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, Long> valueCounts = buildCountsTable(input, builder);
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).shutDownWhenFull())).toStream().to(outputSuppressed, Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().to(outputRaw, Produced.with(STRING_SERDE, Serdes.Long()));
    final Properties streamsConfig = getStreamsConfig(appId);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        verifyErrorShutdown(driver);
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}

Example 15 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressionIntegrationTest method shouldUseDefaultSerdes.

@Test
public void shouldUseDefaultSerdes() {
    final String testId = "-shouldInheritSerdes";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    final KTable<String, String> valueCounts = inputStream.groupByKey().aggregate(() -> "()", (key, value, aggregate) -> aggregate + ",(" + key + ": " + value + ")");
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).emitEarlyWhenFull())).toStream().to(outputSuppressed);
    valueCounts.toStream().to(outputRaw);
    final Properties streamsConfig = getStreamsConfig(appId);
    streamsConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    streamsConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        final boolean rawRecords = waitForAnyRecord(outputRaw);
        final boolean suppressedRecords = waitForAnyRecord(outputSuppressed);
        assertThat(rawRecords, Matchers.is(true));
        assertThat(suppressedRecords, is(true));
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}