Examples with TopologyTestDriver org.apache.kafka.streams.TopologyTestDriver used on opensource projects

Example 11 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class KStreamRepartitionTest method shouldInvokePartitionerWhenSet.

@Test
public void shouldInvokePartitionerWhenSet() {
    final int[] expectedKeys = new int[] { 0, 1 };
    final StreamPartitioner<Integer, String> streamPartitionerMock = EasyMock.mock(StreamPartitioner.class);
    expect(streamPartitionerMock.partition(anyString(), eq(0), eq("X0"), anyInt())).andReturn(1).times(1);
    expect(streamPartitionerMock.partition(anyString(), eq(1), eq("X1"), anyInt())).andReturn(1).times(1);
    replay(streamPartitionerMock);
    final String repartitionOperationName = "test";
    final Repartitioned<Integer, String> repartitioned = Repartitioned.streamPartitioner(streamPartitionerMock).withName(repartitionOperationName);
    builder.<Integer, String>stream(inputTopic).repartition(repartitioned);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<Integer, String> testInputTopic = driver.createInputTopic(inputTopic, new IntegerSerializer(), new StringSerializer());
        final String topicName = repartitionOutputTopic(props, repartitionOperationName);
        final TestOutputTopic<Integer, String> testOutputTopic = driver.createOutputTopic(topicName, new IntegerDeserializer(), new StringDeserializer());
        for (int i = 0; i < 2; i++) {
            testInputTopic.pipeInput(expectedKeys[i], "X" + expectedKeys[i], i + 10);
        }
        assertThat(testOutputTopic.readRecord(), equalTo(new TestRecord<>(0, "X0", Instant.ofEpochMilli(10))));
        assertThat(testOutputTopic.readRecord(), equalTo(new TestRecord<>(1, "X1", Instant.ofEpochMilli(11))));
        assertTrue(testOutputTopic.readRecordsToList().isEmpty());
    }
    verify(streamPartitionerMock);
}

Example 12 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class CogroupedKStreamImplTest method shouldCogroupAndAggregateTwoKStreamsWithSharedKeys.

@Test
public void shouldCogroupAndAggregateTwoKStreamsWithSharedKeys() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
    final KStream<String, String> stream2 = builder.stream("two", stringConsumed);
    final KGroupedStream<String, String> grouped1 = stream1.groupByKey();
    final KGroupedStream<String, String> grouped2 = stream2.groupByKey();
    final KTable<String, String> customers = grouped1.cogroup(STRING_AGGREGATOR).cogroup(grouped2, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER);
    customers.toStream().to(OUTPUT);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> testInputTopic = driver.createInputTopic("one", new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> testInputTopic2 = driver.createInputTopic("two", new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> testOutputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
        testInputTopic.pipeInput("k1", "A", 0L);
        testInputTopic.pipeInput("k2", "A", 1L);
        testInputTopic.pipeInput("k1", "A", 10L);
        testInputTopic.pipeInput("k2", "A", 100L);
        testInputTopic2.pipeInput("k2", "B", 100L);
        testInputTopic2.pipeInput("k2", "B", 200L);
        testInputTopic2.pipeInput("k1", "B", 1L);
        testInputTopic2.pipeInput("k2", "B", 500L);
        testInputTopic2.pipeInput("k1", "B", 500L);
        testInputTopic2.pipeInput("k2", "B", 500L);
        testInputTopic2.pipeInput("k3", "B", 500L);
        testInputTopic2.pipeInput("k2", "B", 100L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "A", 0);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "A", 1);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "AA", 10);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "AA", 100);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "AAB", 100);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "AABB", 200);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "AAB", 10);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "AABBB", 500);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "AABB", 500);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "AABBBB", 500);
    }
}

Example 13 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class CogroupedKStreamImplTest method shouldAllowDifferentOutputTypeInCoGroup.

@Test
public void shouldAllowDifferentOutputTypeInCoGroup() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
    final KStream<String, String> stream2 = builder.stream("two", stringConsumed);
    final KGroupedStream<String, String> grouped1 = stream1.groupByKey();
    final KGroupedStream<String, String> grouped2 = stream2.groupByKey();
    final KTable<String, Integer> customers = grouped1.cogroup(STRING_SUM_AGGREGATOR).cogroup(grouped2, STRING_SUM_AGGREGATOR).aggregate(SUM_INITIALIZER, Materialized.<String, Integer, KeyValueStore<Bytes, byte[]>>as("store1").withValueSerde(Serdes.Integer()));
    customers.toStream().to(OUTPUT);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> testInputTopic = driver.createInputTopic("one", new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> testInputTopic2 = driver.createInputTopic("two", new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, Integer> testOutputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new IntegerDeserializer());
        testInputTopic.pipeInput("k1", "1", 0L);
        testInputTopic.pipeInput("k2", "1", 1L);
        testInputTopic.pipeInput("k1", "1", 10L);
        testInputTopic.pipeInput("k2", "1", 100L);
        testInputTopic2.pipeInput("k2", "2", 100L);
        testInputTopic2.pipeInput("k2", "2", 200L);
        testInputTopic2.pipeInput("k1", "2", 1L);
        testInputTopic2.pipeInput("k2", "2", 500L);
        testInputTopic2.pipeInput("k1", "2", 500L);
        testInputTopic2.pipeInput("k2", "3", 500L);
        testInputTopic2.pipeInput("k3", "2", 500L);
        testInputTopic2.pipeInput("k2", "2", 100L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", 1, 0);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", 1, 1);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", 2, 10);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", 2, 100);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", 4, 100);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", 6, 200);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", 4, 10);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", 8, 500);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", 6, 500);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", 11, 500);
    }
}

Example 14 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class CogroupedKStreamImplTest method testCogroupKeyMixedAggregators.

@Test
public void testCogroupKeyMixedAggregators() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<String, String> stream1 = builder.stream("one", stringConsumed);
    final KStream<String, String> stream2 = builder.stream("two", stringConsumed);
    final KGroupedStream<String, String> grouped1 = stream1.groupByKey();
    final KGroupedStream<String, String> grouped2 = stream2.groupByKey();
    final KTable<String, String> customers = grouped1.cogroup(MockAggregator.TOSTRING_REMOVER).cogroup(grouped2, MockAggregator.TOSTRING_ADDER).aggregate(MockInitializer.STRING_INIT, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as("store1").withValueSerde(Serdes.String()));
    customers.toStream().to(OUTPUT);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> testInputTopic = driver.createInputTopic("one", new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> testInputTopic2 = driver.createInputTopic("two", new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> testOutputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
        testInputTopic.pipeInput("k1", "1", 0L);
        testInputTopic.pipeInput("k2", "1", 1L);
        testInputTopic.pipeInput("k1", "1", 10L);
        testInputTopic.pipeInput("k2", "1", 100L);
        testInputTopic2.pipeInput("k1", "2", 500L);
        testInputTopic2.pipeInput("k2", "2", 500L);
        testInputTopic2.pipeInput("k1", "2", 500L);
        testInputTopic2.pipeInput("k2", "2", 100L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "0-1", 0);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "0-1", 1);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "0-1-1", 10);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "0-1-1", 100);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "0-1-1+2", 500L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "0-1-1+2", 500L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "0-1-1+2+2", 500L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "0-1-1+2+2", 500L);
    }
}

Example 15 with TopologyTestDriver

use of org.apache.kafka.streams.TopologyTestDriver in project kafka by apache.

the class CogroupedKStreamImplTest method testCogroupWithKTableKTableInnerJoin.

@Test
public void testCogroupWithKTableKTableInnerJoin() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KGroupedStream<String, String> grouped1 = builder.stream("one", stringConsumed).groupByKey();
    final KGroupedStream<String, String> grouped2 = builder.stream("two", stringConsumed).groupByKey();
    final KTable<String, String> table1 = grouped1.cogroup(STRING_AGGREGATOR).cogroup(grouped2, STRING_AGGREGATOR).aggregate(STRING_INITIALIZER, Named.as("name"), Materialized.as("store"));
    final KTable<String, String> table2 = builder.table("three", stringConsumed);
    final KTable<String, String> joined = table1.join(table2, MockValueJoiner.TOSTRING_JOINER, Materialized.with(Serdes.String(), Serdes.String()));
    joined.toStream().to(OUTPUT);
    try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), props)) {
        final TestInputTopic<String, String> testInputTopic = driver.createInputTopic("one", new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> testInputTopic2 = driver.createInputTopic("two", new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> testInputTopic3 = driver.createInputTopic("three", new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> testOutputTopic = driver.createOutputTopic(OUTPUT, new StringDeserializer(), new StringDeserializer());
        testInputTopic.pipeInput("k1", "A", 5L);
        testInputTopic2.pipeInput("k2", "B", 6L);
        assertTrue(testOutputTopic.isEmpty());
        testInputTopic3.pipeInput("k1", "C", 0L);
        testInputTopic3.pipeInput("k2", "D", 10L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k1", "A+C", 5L);
        assertOutputKeyValueTimestamp(testOutputTopic, "k2", "B+D", 10L);
        assertTrue(testOutputTopic.isEmpty());
    }
}