Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 51 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamTransformValuesTest method testTransform.

// Old PAPI. Needs to be migrated.
@SuppressWarnings("deprecation")
@Test
public void testTransform() {
    final StreamsBuilder builder = new StreamsBuilder();
    final ValueTransformerSupplier<Number, Integer> valueTransformerSupplier = () -> new ValueTransformer<Number, Integer>() {

        private int total = 0;

        @Override
        public void init(final org.apache.kafka.streams.processor.ProcessorContext context) {
        }

        @Override
        public Integer transform(final Number value) {
            total += value.intValue();
            return total;
        }

        @Override
        public void close() {
        }
    };
    final int[] expectedKeys = { 1, 10, 100, 1000 };
    final KStream<Integer, Integer> stream;
    stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    stream.transformValues(valueTransformerSupplier).process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        for (final int expectedKey : expectedKeys) {
            final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer());
            inputTopic.pipeInput(expectedKey, expectedKey * 10, expectedKey / 2L);
        }
    }
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(1, 10, 0), new KeyValueTimestamp<>(10, 110, 5), new KeyValueTimestamp<>(100, 1110, 50), new KeyValueTimestamp<>(1000, 11110, 500) };
    assertArrayEquals(expected, supplier.theCapturedProcessor().processed().toArray());
}

Example 52 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamTransformValuesTest method testTransformWithKey.

// Old PAPI. Needs to be migrated.
@SuppressWarnings("deprecation")
@Test
public void testTransformWithKey() {
    final StreamsBuilder builder = new StreamsBuilder();
    final ValueTransformerWithKeySupplier<Integer, Number, Integer> valueTransformerSupplier = () -> new ValueTransformerWithKey<Integer, Number, Integer>() {

        private int total = 0;

        @Override
        public void init(final org.apache.kafka.streams.processor.ProcessorContext context) {
        }

        @Override
        public Integer transform(final Integer readOnlyKey, final Number value) {
            total += value.intValue() + readOnlyKey;
            return total;
        }

        @Override
        public void close() {
        }
    };
    final int[] expectedKeys = { 1, 10, 100, 1000 };
    final KStream<Integer, Integer> stream;
    stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    stream.transformValues(valueTransformerSupplier).process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer());
        for (final int expectedKey : expectedKeys) {
            inputTopic.pipeInput(expectedKey, expectedKey * 10, expectedKey / 2L);
        }
    }
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(1, 11, 0), new KeyValueTimestamp<>(10, 121, 5), new KeyValueTimestamp<>(100, 1221, 50), new KeyValueTimestamp<>(1000, 12221, 500) };
    assertArrayEquals(expected, supplier.theCapturedProcessor().processed().toArray());
}

Example 53 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressionDurabilityIntegrationTest method shouldRecoverBufferAfterShutdown.

@Test
public void shouldRecoverBufferAfterShutdown() {
    final String testId = safeUniqueTestName(getClass(), testName);
    final String appId = "appId_" + testId;
    final String input = "input" + testId;
    final String storeName = "counts";
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    // create multiple partitions as a trap, in case the buffer doesn't properly set the
    // partition on the records, but instead relies on the default key partitioner
    cleanStateBeforeTest(CLUSTER, 2, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, Long> valueCounts = builder.stream(input, Consumed.with(STRING_SERDE, STRING_SERDE)).groupByKey().count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(storeName).withCachingDisabled());
    final KStream<String, Long> suppressedCounts = valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(3L).emitEarlyWhenFull())).toStream();
    final AtomicInteger eventCount = new AtomicInteger(0);
    suppressedCounts.foreach((key, value) -> eventCount.incrementAndGet());
    // expect all post-suppress records to keep the right input topic
    final MetadataValidator metadataValidator = new MetadataValidator(input);
    suppressedCounts.transform(metadataValidator).to(outputSuppressed, Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().transform(metadataValidator).to(outputRaw, Produced.with(STRING_SERDE, Serdes.Long()));
    final Properties streamsConfig = mkProperties(mkMap(mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, appId), mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()), mkEntry(StreamsConfig.POLL_MS_CONFIG, Long.toString(COMMIT_INTERVAL)), mkEntry(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, processingGuaranteee), mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath())));
    streamsConfig.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, COMMIT_INTERVAL);
    KafkaStreams driver = getStartedStreams(streamsConfig, builder, true);
    try {
        // start by putting some stuff in the buffer
        // note, we send all input records to partition 0
        // to make sure that supppress doesn't erroneously send records to other partitions.
        produceSynchronouslyToPartitionZero(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v2", scaledTime(2L)), new KeyValueTimestamp<>("k3", "v3", scaledTime(3L))));
        verifyOutput(outputRaw, new HashSet<>(asList(new KeyValueTimestamp<>("k1", 1L, scaledTime(1L)), new KeyValueTimestamp<>("k2", 1L, scaledTime(2L)), new KeyValueTimestamp<>("k3", 1L, scaledTime(3L)))));
        assertThat(eventCount.get(), is(0));
        // flush two of the first three events out.
        produceSynchronouslyToPartitionZero(input, asList(new KeyValueTimestamp<>("k4", "v4", scaledTime(4L)), new KeyValueTimestamp<>("k5", "v5", scaledTime(5L))));
        verifyOutput(outputRaw, new HashSet<>(asList(new KeyValueTimestamp<>("k4", 1L, scaledTime(4L)), new KeyValueTimestamp<>("k5", 1L, scaledTime(5L)))));
        assertThat(eventCount.get(), is(2));
        verifyOutput(outputSuppressed, asList(new KeyValueTimestamp<>("k1", 1L, scaledTime(1L)), new KeyValueTimestamp<>("k2", 1L, scaledTime(2L))));
        // bounce to ensure that the history, including retractions,
        // get restored properly. (i.e., we shouldn't see those first events again)
        // restart the driver
        driver.close();
        assertThat(driver.state(), is(KafkaStreams.State.NOT_RUNNING));
        driver = getStartedStreams(streamsConfig, builder, false);
        // flush those recovered buffered events out.
        produceSynchronouslyToPartitionZero(input, asList(new KeyValueTimestamp<>("k6", "v6", scaledTime(6L)), new KeyValueTimestamp<>("k7", "v7", scaledTime(7L)), new KeyValueTimestamp<>("k8", "v8", scaledTime(8L))));
        verifyOutput(outputRaw, new HashSet<>(asList(new KeyValueTimestamp<>("k6", 1L, scaledTime(6L)), new KeyValueTimestamp<>("k7", 1L, scaledTime(7L)), new KeyValueTimestamp<>("k8", 1L, scaledTime(8L)))));
        assertThat("suppress has apparently produced some duplicates. There should only be 5 output events.", eventCount.get(), is(5));
        verifyOutput(outputSuppressed, asList(new KeyValueTimestamp<>("k3", 1L, scaledTime(3L)), new KeyValueTimestamp<>("k4", 1L, scaledTime(4L)), new KeyValueTimestamp<>("k5", 1L, scaledTime(5L))));
        metadataValidator.raiseExceptionIfAny();
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}

Example 54 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressionIntegrationTest method shouldCreateChangelogByDefault.

@Test
public void shouldCreateChangelogByDefault() {
    final String testId = "-shouldCreateChangelogByDefault";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    final String changeLog = "suppressionintegrationtest-shouldCreateChangelogByDefault-KTABLE-SUPPRESS-STATE-STORE-0000000004-changelog";
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    final KTable<String, String> valueCounts = inputStream.groupByKey().aggregate(() -> "()", (key, value, aggregate) -> aggregate + ",(" + key + ": " + value + ")");
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).emitEarlyWhenFull())).toStream().to(outputSuppressed);
    valueCounts.toStream().to(outputRaw);
    final Properties streamsConfig = getStreamsConfig(appId);
    streamsConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    streamsConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        final boolean rawRecords = waitForAnyRecord(outputRaw);
        final boolean suppressedRecords = waitForAnyRecord(outputSuppressed);
        assertThat(CLUSTER.getAllTopicsInCluster(), hasItem(changeLog));
        assertThat(rawRecords, Matchers.is(true));
        assertThat(suppressedRecords, is(true));
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}

Example 55 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressionIntegrationTest method shouldAllowOverridingChangelogConfig.

@Test
public void shouldAllowOverridingChangelogConfig() {
    final String testId = "-shouldAllowOverridingChangelogConfig";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    final Map<String, String> logConfig = Collections.singletonMap("retention.ms", "1000");
    final String changeLog = "suppressionintegrationtest-shouldAllowOverridingChangelogConfig-KTABLE-SUPPRESS-STATE-STORE-0000000004-changelog";
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    final KTable<String, String> valueCounts = inputStream.groupByKey().aggregate(() -> "()", (key, value, aggregate) -> aggregate + ",(" + key + ": " + value + ")");
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).emitEarlyWhenFull().withLoggingEnabled(logConfig))).toStream().to(outputSuppressed);
    valueCounts.toStream().to(outputRaw);
    final Properties streamsConfig = getStreamsConfig(appId);
    streamsConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    streamsConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        final boolean rawRecords = waitForAnyRecord(outputRaw);
        final boolean suppressedRecords = waitForAnyRecord(outputSuppressed);
        final Properties config = CLUSTER.getLogConfig(changeLog);
        assertThat(config.getProperty("retention.ms"), is(logConfig.get("retention.ms")));
        assertThat(CLUSTER.getAllTopicsInCluster(), hasItem(changeLog));
        assertThat(rawRecords, Matchers.is(true));
        assertThat(suppressedRecords, is(true));
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}