Examples with TestInputTopic org.apache.kafka.streams.TestInputTopic used on opensource projects

Example 21 with TestInputTopic

use of org.apache.kafka.streams.TestInputTopic in project kafka by apache.

the class KStreamFlatMapValuesTest method testFlatMapValues.

@Test
public void testFlatMapValues() {
    final StreamsBuilder builder = new StreamsBuilder();
    final ValueMapper<Number, Iterable<String>> mapper = value -> {
        final ArrayList<String> result = new ArrayList<>();
        result.add("v" + value);
        result.add("V" + value);
        return result;
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    final KStream<Integer, Integer> stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    stream.flatMapValues(mapper).process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        for (final int expectedKey : expectedKeys) {
            // passing the timestamp to inputTopic.create to disambiguate the call
            inputTopic.pipeInput(expectedKey, expectedKey, 0L);
        }
    }
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(0, "v0", 0), new KeyValueTimestamp<>(0, "V0", 0), new KeyValueTimestamp<>(1, "v1", 0), new KeyValueTimestamp<>(1, "V1", 0), new KeyValueTimestamp<>(2, "v2", 0), new KeyValueTimestamp<>(2, "V2", 0), new KeyValueTimestamp<>(3, "v3", 0), new KeyValueTimestamp<>(3, "V3", 0) };
    assertArrayEquals(expected, supplier.theCapturedProcessor().processed().toArray());
}

Example 22 with TestInputTopic

use of org.apache.kafka.streams.TestInputTopic in project kafka by apache.

the class KStreamFlatMapValuesTest method testFlatMapValuesWithKeys.

@Test
public void testFlatMapValuesWithKeys() {
    final StreamsBuilder builder = new StreamsBuilder();
    final ValueMapperWithKey<Integer, Number, Iterable<String>> mapper = (readOnlyKey, value) -> {
        final ArrayList<String> result = new ArrayList<>();
        result.add("v" + value);
        result.add("k" + readOnlyKey);
        return result;
    };
    final int[] expectedKeys = { 0, 1, 2, 3 };
    final KStream<Integer, Integer> stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.Integer()));
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    stream.flatMapValues(mapper).process(supplier);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, Integer> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        for (final int expectedKey : expectedKeys) {
            // passing the timestamp to inputTopic.create to disambiguate the call
            inputTopic.pipeInput(expectedKey, expectedKey, 0L);
        }
    }
    final KeyValueTimestamp[] expected = { new KeyValueTimestamp<>(0, "v0", 0), new KeyValueTimestamp<>(0, "k0", 0), new KeyValueTimestamp<>(1, "v1", 0), new KeyValueTimestamp<>(1, "k1", 0), new KeyValueTimestamp<>(2, "v2", 0), new KeyValueTimestamp<>(2, "k2", 0), new KeyValueTimestamp<>(3, "v3", 0), new KeyValueTimestamp<>(3, "k3", 0) };
    assertArrayEquals(expected, supplier.theCapturedProcessor().processed().toArray());
}

Example 23 with TestInputTopic

use of org.apache.kafka.streams.TestInputTopic in project kafka by apache.

the class KStreamForeachTest method testForeach.

@Test
public void testForeach() {
    // Given
    final List<KeyValue<Integer, String>> inputRecords = Arrays.asList(new KeyValue<>(0, "zero"), new KeyValue<>(1, "one"), new KeyValue<>(2, "two"), new KeyValue<>(3, "three"));
    final List<KeyValue<Integer, String>> expectedRecords = Arrays.asList(new KeyValue<>(0, "ZERO"), new KeyValue<>(2, "ONE"), new KeyValue<>(4, "TWO"), new KeyValue<>(6, "THREE"));
    final List<KeyValue<Integer, String>> actualRecords = new ArrayList<>();
    final ForeachAction<Integer, String> action = (key, value) -> actualRecords.add(new KeyValue<>(key * 2, value.toUpperCase(Locale.ROOT)));
    // When
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, String> stream = builder.stream(topicName, Consumed.with(Serdes.Integer(), Serdes.String()));
    stream.foreach(action);
    // Then
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, String> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
        for (final KeyValue<Integer, String> record : inputRecords) {
            inputTopic.pipeInput(record.key, record.value);
        }
    }
    assertEquals(expectedRecords.size(), actualRecords.size());
    for (int i = 0; i < expectedRecords.size(); i++) {
        final KeyValue<Integer, String> expectedRecord = expectedRecords.get(i);
        final KeyValue<Integer, String> actualRecord = actualRecords.get(i);
        assertEquals(expectedRecord, actualRecord);
    }
}

Example 24 with TestInputTopic

use of org.apache.kafka.streams.TestInputTopic in project kafka by apache.

the class SuppressScenarioTest method shouldImmediatelyEmitEventsWithZeroEmitAfter.

@Test
public void shouldImmediatelyEmitEventsWithZeroEmitAfter() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, Long> valueCounts = builder.table("input", Consumed.with(STRING_SERDE, STRING_SERDE), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>with(STRING_SERDE, STRING_SERDE).withCachingDisabled().withLoggingDisabled()).groupBy((k, v) -> new KeyValue<>(v, k), Grouped.with(STRING_SERDE, STRING_SERDE)).count();
    valueCounts.suppress(untilTimeLimit(ZERO, unbounded())).toStream().to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
    final Topology topology = builder.build();
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
        inputTopic.pipeInput("k1", "v1", 0L);
        inputTopic.pipeInput("k1", "v2", 1L);
        inputTopic.pipeInput("k2", "v1", 2L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("v1", 1L, 0L), new KeyValueTimestamp<>("v1", 0L, 1L), new KeyValueTimestamp<>("v2", 1L, 1L), new KeyValueTimestamp<>("v1", 1L, 2L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("v1", 1L, 0L), new KeyValueTimestamp<>("v1", 0L, 1L), new KeyValueTimestamp<>("v2", 1L, 1L), new KeyValueTimestamp<>("v1", 1L, 2L)));
        inputTopic.pipeInput("x", "x", 3L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("x", 1L, 3L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("x", 1L, 3L)));
        inputTopic.pipeInput("x", "y", 4L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("x", 0L, 4L), new KeyValueTimestamp<>("y", 1L, 4L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("x", 0L, 4L), new KeyValueTimestamp<>("y", 1L, 4L)));
    }
}

Example 25 with TestInputTopic

use of org.apache.kafka.streams.TestInputTopic in project kafka by apache.

the class SuppressScenarioTest method shouldSuppressIntermediateEventsWithTimeLimit.

@Test
public void shouldSuppressIntermediateEventsWithTimeLimit() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<String, Long> valueCounts = builder.table("input", Consumed.with(STRING_SERDE, STRING_SERDE), Materialized.<String, String, KeyValueStore<Bytes, byte[]>>with(STRING_SERDE, STRING_SERDE).withCachingDisabled().withLoggingDisabled()).groupBy((k, v) -> new KeyValue<>(v, k), Grouped.with(STRING_SERDE, STRING_SERDE)).count();
    valueCounts.suppress(untilTimeLimit(ofMillis(2L), unbounded())).toStream().to("output-suppressed", Produced.with(STRING_SERDE, Serdes.Long()));
    valueCounts.toStream().to("output-raw", Produced.with(STRING_SERDE, Serdes.Long()));
    final Topology topology = builder.build();
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, config)) {
        final TestInputTopic<String, String> inputTopic = driver.createInputTopic("input", STRING_SERIALIZER, STRING_SERIALIZER);
        inputTopic.pipeInput("k1", "v1", 0L);
        inputTopic.pipeInput("k1", "v2", 1L);
        inputTopic.pipeInput("k2", "v1", 2L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("v1", 1L, 0L), new KeyValueTimestamp<>("v1", 0L, 1L), new KeyValueTimestamp<>("v2", 1L, 1L), new KeyValueTimestamp<>("v1", 1L, 2L)));
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("v1", 1L, 2L)));
        // inserting a dummy "tick" record just to advance stream time
        inputTopic.pipeInput("tick", "tick", 3L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("tick", 1L, 3L)));
        // the stream time is now 3, so it's time to emit this record
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), singletonList(new KeyValueTimestamp<>("v2", 1L, 1L)));
        inputTopic.pipeInput("tick", "tock", 4L);
        verify(drainProducerRecords(driver, "output-raw", STRING_DESERIALIZER, LONG_DESERIALIZER), asList(new KeyValueTimestamp<>("tick", 0L, 4L), new KeyValueTimestamp<>("tock", 1L, 4L)));
        // tick is still buffered, since it was first inserted at time 3, and it is only time 4 right now.
        verify(drainProducerRecords(driver, "output-suppressed", STRING_DESERIALIZER, LONG_DESERIALIZER), emptyList());
    }
}