Examples with Topology org.apache.kafka.streams.Topology used on opensource projects

Example 36 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableFilterTest method doTestSendingOldValue.

private void doTestSendingOldValue(final StreamsBuilder builder, final KTableImpl<String, Integer, Integer> table1, final KTableImpl<String, Integer, Integer> table2, final String topic1) {
    final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = new MockApiProcessorSupplier<>();
    final Topology topology = builder.build();
    topology.addProcessor("proc1", supplier, table1.name);
    topology.addProcessor("proc2", supplier, table2.name);
    final boolean parentSendOldVals = table1.sendingOldValueEnabled();
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<String, Integer> inputTopic = driver.createInputTopic(topic1, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        inputTopic.pipeInput("A", 1, 5L);
        inputTopic.pipeInput("B", 1, 10L);
        inputTopic.pipeInput("C", 1, 15L);
        final List<MockApiProcessor<String, Integer, Void, Void>> processors = supplier.capturedProcessors(2);
        final MockApiProcessor<String, Integer, Void, Void> table1Output = processors.get(0);
        final MockApiProcessor<String, Integer, Void, Void> table2Output = processors.get(1);
        table1Output.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(1, null), 5), new KeyValueTimestamp<>("B", new Change<>(1, null), 10), new KeyValueTimestamp<>("C", new Change<>(1, null), 15));
        table2Output.checkEmptyAndClearProcessResult();
        inputTopic.pipeInput("A", 2, 15L);
        inputTopic.pipeInput("B", 2, 8L);
        table1Output.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, parentSendOldVals ? 1 : null), 15), new KeyValueTimestamp<>("B", new Change<>(2, parentSendOldVals ? 1 : null), 8));
        table2Output.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(2, null), 15), new KeyValueTimestamp<>("B", new Change<>(2, null), 8));
        inputTopic.pipeInput("A", 3, 20L);
        table1Output.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(3, parentSendOldVals ? 2 : null), 20));
        table2Output.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, 2), 20));
        inputTopic.pipeInput("A", null, 10L);
        inputTopic.pipeInput("B", null, 20L);
        table1Output.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, parentSendOldVals ? 3 : null), 10), new KeyValueTimestamp<>("B", new Change<>(null, parentSendOldVals ? 2 : null), 20));
        table2Output.checkAndClearProcessResult(new KeyValueTimestamp<>("B", new Change<>(null, 2), 20));
    }
}

Example 37 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableFilterTest method doTestSkipNullOnMaterialization.

private void doTestSkipNullOnMaterialization(final StreamsBuilder builder, final KTableImpl<String, String, String> table1, final KTableImpl<String, String, String> table2, final String topic1) {
    final MockApiProcessorSupplier<String, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    final Topology topology = builder.build();
    topology.addProcessor("proc1", supplier, table1.name);
    topology.addProcessor("proc2", supplier, table2.name);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<String, String> stringinputTopic = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        stringinputTopic.pipeInput("A", "reject", 5L);
        stringinputTopic.pipeInput("B", "reject", 10L);
        stringinputTopic.pipeInput("C", "reject", 20L);
    }
    final List<MockApiProcessor<String, String, Void, Void>> processors = supplier.capturedProcessors(2);
    processors.get(0).checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("reject", null), 5), new KeyValueTimestamp<>("B", new Change<>("reject", null), 10), new KeyValueTimestamp<>("C", new Change<>("reject", null), 20));
    processors.get(1).checkEmptyAndClearProcessResult();
}

Example 38 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableImplTest method shouldCreateSourceAndSinkNodesForRepartitioningTopic.

@Test
public void shouldCreateSourceAndSinkNodesForRepartitioningTopic() throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    final String storeName1 = "storeName1";
    final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, consumed, Materialized.<String, String, KeyValueStore<Bytes, byte[]>>as(storeName1).withKeySerde(Serdes.String()).withValueSerde(Serdes.String()));
    table1.groupBy(MockMapper.noOpKeyValueMapper()).aggregate(MockInitializer.STRING_INIT, MockAggregator.TOSTRING_ADDER, MockAggregator.TOSTRING_REMOVER, Materialized.as("mock-result1"));
    table1.groupBy(MockMapper.noOpKeyValueMapper()).reduce(MockReducer.STRING_ADDER, MockReducer.STRING_REMOVER, Materialized.as("mock-result2"));
    final Topology topology = builder.build();
    try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(topology, props)) {
        assertEquals(3, driver.getAllStateStores().size());
        assertTopologyContainsProcessor(topology, "KSTREAM-SINK-0000000003");
        assertTopologyContainsProcessor(topology, "KSTREAM-SOURCE-0000000004");
        assertTopologyContainsProcessor(topology, "KSTREAM-SINK-0000000007");
        assertTopologyContainsProcessor(topology, "KSTREAM-SOURCE-0000000008");
        final Field valSerializerField = ((SinkNode) driver.getProcessor("KSTREAM-SINK-0000000003")).getClass().getDeclaredField("valSerializer");
        final Field valDeserializerField = ((SourceNode) driver.getProcessor("KSTREAM-SOURCE-0000000004")).getClass().getDeclaredField("valDeserializer");
        valSerializerField.setAccessible(true);
        valDeserializerField.setAccessible(true);
        assertNotNull(((ChangedSerializer) valSerializerField.get(driver.getProcessor("KSTREAM-SINK-0000000003"))).inner());
        assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.getProcessor("KSTREAM-SOURCE-0000000004"))).inner());
        assertNotNull(((ChangedSerializer) valSerializerField.get(driver.getProcessor("KSTREAM-SINK-0000000007"))).inner());
        assertNotNull(((ChangedDeserializer) valDeserializerField.get(driver.getProcessor("KSTREAM-SOURCE-0000000008"))).inner());
    }
}

Example 39 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableSourceTest method testValueGetter.

@Test
public void testValueGetter() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    @SuppressWarnings("unchecked") final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, stringConsumed, Materialized.as("store"));
    final Topology topology = builder.build();
    final KTableValueGetterSupplier<String, String> getterSupplier1 = table1.valueGetterSupplier();
    final InternalTopologyBuilder topologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology);
    topologyBuilder.connectProcessorAndStateStores(table1.name, getterSupplier1.storeNames());
    try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(builder.build(), props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final KTableValueGetter<String, String> getter1 = getterSupplier1.get();
        getter1.init(driver.setCurrentNodeForProcessorContext(table1.name));
        inputTopic1.pipeInput("A", "01", 10L);
        inputTopic1.pipeInput("B", "01", 20L);
        inputTopic1.pipeInput("C", "01", 15L);
        assertEquals(ValueAndTimestamp.make("01", 10L), getter1.get("A"));
        assertEquals(ValueAndTimestamp.make("01", 20L), getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
        inputTopic1.pipeInput("A", "02", 30L);
        inputTopic1.pipeInput("B", "02", 5L);
        assertEquals(ValueAndTimestamp.make("02", 30L), getter1.get("A"));
        assertEquals(ValueAndTimestamp.make("02", 5L), getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
        inputTopic1.pipeInput("A", "03", 29L);
        assertEquals(ValueAndTimestamp.make("03", 29L), getter1.get("A"));
        assertEquals(ValueAndTimestamp.make("02", 5L), getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
        inputTopic1.pipeInput("A", null, 50L);
        inputTopic1.pipeInput("B", null, 3L);
        assertNull(getter1.get("A"));
        assertNull(getter1.get("B"));
        assertEquals(ValueAndTimestamp.make("01", 15L), getter1.get("C"));
    }
}

Example 40 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableSourceTest method testSendingOldValue.

@Test
public void testSendingOldValue() {
    final StreamsBuilder builder = new StreamsBuilder();
    final String topic1 = "topic1";
    @SuppressWarnings("unchecked") final KTableImpl<String, String, String> table1 = (KTableImpl<String, String, String>) builder.table(topic1, stringConsumed);
    table1.enableSendingOldValues(true);
    assertTrue(table1.sendingOldValueEnabled());
    final MockApiProcessorSupplier<String, Integer, Void, Void> supplier = new MockApiProcessorSupplier<>();
    final Topology topology = builder.build().addProcessor("proc1", supplier, table1.name);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<String, String> inputTopic1 = driver.createInputTopic(topic1, new StringSerializer(), new StringSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final MockApiProcessor<String, Integer, Void, Void> proc1 = supplier.theCapturedProcessor();
        inputTopic1.pipeInput("A", "01", 10L);
        inputTopic1.pipeInput("B", "01", 20L);
        inputTopic1.pipeInput("C", "01", 15L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("01", null), 10), new KeyValueTimestamp<>("B", new Change<>("01", null), 20), new KeyValueTimestamp<>("C", new Change<>("01", null), 15));
        inputTopic1.pipeInput("A", "02", 8L);
        inputTopic1.pipeInput("B", "02", 22L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("02", "01"), 8), new KeyValueTimestamp<>("B", new Change<>("02", "01"), 22));
        inputTopic1.pipeInput("A", "03", 12L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>("03", "02"), 12));
        inputTopic1.pipeInput("A", null, 15L);
        inputTopic1.pipeInput("B", null, 20L);
        proc1.checkAndClearProcessResult(new KeyValueTimestamp<>("A", new Change<>(null, "03"), 15), new KeyValueTimestamp<>("B", new Change<>(null, "02"), 20));
    }
}