Examples with KeyValueTimestamp org.apache.kafka.streams.KeyValueTimestamp used on opensource projects

Example 56 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class SuppressionIntegrationTest method shouldAllowDisablingChangelog.

@Test
public void shouldAllowDisablingChangelog() {
    final String testId = "-shouldAllowDisablingChangelog";
    final String appId = getClass().getSimpleName().toLowerCase(Locale.getDefault()) + testId;
    final String input = "input" + testId;
    final String outputSuppressed = "output-suppressed" + testId;
    final String outputRaw = "output-raw" + testId;
    cleanStateBeforeTest(CLUSTER, input, outputRaw, outputSuppressed);
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> inputStream = builder.stream(input);
    final KTable<String, String> valueCounts = inputStream.groupByKey().aggregate(() -> "()", (key, value, aggregate) -> aggregate + ",(" + key + ": " + value + ")");
    valueCounts.suppress(untilTimeLimit(ofMillis(MAX_VALUE), maxRecords(1L).emitEarlyWhenFull().withLoggingDisabled())).toStream().to(outputSuppressed);
    valueCounts.toStream().to(outputRaw);
    final Properties streamsConfig = getStreamsConfig(appId);
    streamsConfig.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    streamsConfig.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    final KafkaStreams driver = IntegrationTestUtils.getStartedStreams(streamsConfig, builder, true);
    try {
        produceSynchronously(input, asList(new KeyValueTimestamp<>("k1", "v1", scaledTime(0L)), new KeyValueTimestamp<>("k1", "v2", scaledTime(1L)), new KeyValueTimestamp<>("k2", "v1", scaledTime(2L)), new KeyValueTimestamp<>("x", "x", scaledTime(3L))));
        final boolean rawRecords = waitForAnyRecord(outputRaw);
        final boolean suppressedRecords = waitForAnyRecord(outputSuppressed);
        final Set<String> suppressChangeLog = CLUSTER.getAllTopicsInCluster().stream().filter(s -> s.contains("-changelog")).filter(s -> s.contains("KTABLE-SUPPRESS")).collect(Collectors.toSet());
        assertThat(suppressChangeLog, is(empty()));
        assertThat(rawRecords, Matchers.is(true));
        assertThat(suppressedRecords, is(true));
    } finally {
        driver.close();
        quietlyCleanStateAfterTest(CLUSTER, driver);
    }
}

Example 57 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class IntegrationTestUtils method waitUntilFinalKeyValueRecordsReceived.

@SuppressWarnings("unchecked")
private static <K, V, T> List<T> waitUntilFinalKeyValueRecordsReceived(final Properties consumerConfig, final String topic, final List<T> expectedRecords, final long waitTime, final boolean withTimestamp) throws Exception {
    final List<T> accumData = new ArrayList<>();
    try (final Consumer<K, V> consumer = createConsumer(consumerConfig)) {
        final TestCondition valuesRead = () -> {
            final List<T> readData;
            if (withTimestamp) {
                readData = (List<T>) readKeyValuesWithTimestamp(topic, consumer, waitTime, expectedRecords.size());
            } else {
                readData = (List<T>) readKeyValues(topic, consumer, waitTime, expectedRecords.size());
            }
            accumData.addAll(readData);
            // filter out all intermediate records we don't want
            final List<T> accumulatedActual = accumData.stream().filter(expectedRecords::contains).collect(Collectors.toList());
            // still need to check that for each key, the ordering is expected
            final Map<K, List<T>> finalAccumData = new HashMap<>();
            for (final T kv : accumulatedActual) {
                finalAccumData.computeIfAbsent(withTimestamp ? ((KeyValueTimestamp<K, V>) kv).key() : ((KeyValue<K, V>) kv).key, key -> new ArrayList<>()).add(kv);
            }
            final Map<K, List<T>> finalExpected = new HashMap<>();
            for (final T kv : expectedRecords) {
                finalExpected.computeIfAbsent(withTimestamp ? ((KeyValueTimestamp<K, V>) kv).key() : ((KeyValue<K, V>) kv).key, key -> new ArrayList<>()).add(kv);
            }
            // and the last record received matches the last expected record
            return finalAccumData.equals(finalExpected);
        };
        final String conditionDetails = "Did not receive all " + expectedRecords + " records from topic " + topic + " (got " + accumData + ")";
        TestUtils.waitForCondition(valuesRead, waitTime, conditionDetails);
    }
    return accumData;
}

Example 58 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamImplTest method shouldProcessFromSourceThatMatchPattern.

@Test
public void shouldProcessFromSourceThatMatchPattern() {
    final KStream<String, String> pattern2Source = builder.stream(Pattern.compile("topic-\\d"));
    pattern2Source.process(processorSupplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic3 = driver.createInputTopic("topic-3", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopic4 = driver.createInputTopic("topic-4", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopic5 = driver.createInputTopic("topic-5", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopic6 = driver.createInputTopic("topic-6", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<String, String> inputTopic7 = driver.createInputTopic("topic-7", new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        inputTopic3.pipeInput("A", "aa", 1L);
        inputTopic4.pipeInput("B", "bb", 5L);
        inputTopic5.pipeInput("C", "cc", 10L);
        inputTopic6.pipeInput("D", "dd", 8L);
        inputTopic7.pipeInput("E", "ee", 3L);
    }
    assertEquals(asList(new KeyValueTimestamp<>("A", "aa", 1), new KeyValueTimestamp<>("B", "bb", 5), new KeyValueTimestamp<>("C", "cc", 10), new KeyValueTimestamp<>("D", "dd", 8), new KeyValueTimestamp<>("E", "ee", 3)), processorSupplier.theCapturedProcessor().processed());
}

Example 59 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamFlatMapValuesTest method testFlatMapValues.

@Test
public void testFlatMapValues() {
    final StreamsBuilder builder = new StreamsBuilder();
    final ValueMapper<Number, Iterable<String>> mapper = value -> {
        final ArrayList<String> result = new ArrayList<>();
        result.add("v" + value);
        result.add("V" + value);
        return result;
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    final KStream<Integer, Integer> stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    stream.flatMapValues(mapper).process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        for (final int expectedKey : expectedKeys) {
            // passing the timestamp to inputTopic.create to disambiguate the call
            inputTopic.pipeInput(expectedKey, expectedKey, 0L);
        }
    }
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(0, "v0", 0), new KeyValueTimestamp<>(0, "V0", 0), new KeyValueTimestamp<>(1, "v1", 0), new KeyValueTimestamp<>(1, "V1", 0), new KeyValueTimestamp<>(2, "v2", 0), new KeyValueTimestamp<>(2, "V2", 0), new KeyValueTimestamp<>(3, "v3", 0), new KeyValueTimestamp<>(3, "V3", 0) };
    assertArrayEquals(expected, supplier.theCapturedProcessor().processed().toArray());
}

Example 60 with KeyValueTimestamp

use of org.apache.kafka.streams.KeyValueTimestamp in project kafka by apache.

the class KStreamFlatMapValuesTest method testFlatMapValuesWithKeys.

@Test
public void testFlatMapValuesWithKeys() {
    final StreamsBuilder builder = new StreamsBuilder();
    final ValueMapperWithKey<Integer, Number, Iterable<String>> mapper = (readOnlyKey, value) -> {
        final ArrayList<String> result = new ArrayList<>();
        result.add("v" + value);
        result.add("k" + readOnlyKey);
        return result;
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    final KStream<Integer, Integer> stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    stream.flatMapValues(mapper).process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        for (final int expectedKey : expectedKeys) {
            // passing the timestamp to inputTopic.create to disambiguate the call
            inputTopic.pipeInput(expectedKey, expectedKey, 0L);
        }
    }
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(0, "v0", 0), new KeyValueTimestamp<>(0, "k0", 0), new KeyValueTimestamp<>(1, "v1", 0), new KeyValueTimestamp<>(1, "k1", 0), new KeyValueTimestamp<>(2, "v2", 0), new KeyValueTimestamp<>(2, "k2", 0), new KeyValueTimestamp<>(3, "v3", 0), new KeyValueTimestamp<>(3, "k3", 0) };
    assertArrayEquals(expected, supplier.theCapturedProcessor().processed().toArray());
}