Examples with Topology org.apache.kafka.streams.Topology used on opensource projects

Example 91 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KStreamKTableJoinTest method shouldReuseRepartitionTopicWithGeneratedName.

@Test
public void shouldReuseRepartitionTopicWithGeneratedName() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties props = new Properties();
    props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.NO_OPTIMIZATION);
    final KStream<String, String> streamA = builder.stream("topic", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableB = builder.table("topic2", Consumed.with(Serdes.String(), Serdes.String()));
    final KTable<String, String> tableC = builder.table("topic3", Consumed.with(Serdes.String(), Serdes.String()));
    final KStream<String, String> rekeyedStream = streamA.map((k, v) -> new KeyValue<>(v, k));
    rekeyedStream.join(tableB, (value1, value2) -> value1 + value2).to("out-one");
    rekeyedStream.join(tableC, (value1, value2) -> value1 + value2).to("out-two");
    final Topology topology = builder.build(props);
    assertEquals(expectedTopologyWithGeneratedRepartitionTopicNames, topology.describe().toString());
}

Example 92 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableFilterTest method doTestValueGetter.

private void doTestValueGetter(final StreamsBuilder builder, final KTableImpl<String, Integer, Integer> table2, final KTableImpl<String, Integer, Integer> table3, final String topic1) {
    final Topology topology = builder.build();
    final KTableValueGetterSupplier<String, Integer> getterSupplier2 = table2.valueGetterSupplier();
    final KTableValueGetterSupplier<String, Integer> getterSupplier3 = table3.valueGetterSupplier();
    final InternalTopologyBuilder topologyBuilder = TopologyWrapper.getInternalTopologyBuilder(topology);
    topologyBuilder.connectProcessorAndStateStores(table2.name, getterSupplier2.storeNames());
    topologyBuilder.connectProcessorAndStateStores(table3.name, getterSupplier3.storeNames());
    try (final TopologyTestDriverWrapper driver = new TopologyTestDriverWrapper(topology, props)) {
        final TestInputTopic<String, Integer> inputTopic = driver.createInputTopic(topic1, new StringSerializer(), new IntegerSerializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final KTableValueGetter<String, Integer> getter2 = getterSupplier2.get();
        final KTableValueGetter<String, Integer> getter3 = getterSupplier3.get();
        getter2.init(driver.setCurrentNodeForProcessorContext(table2.name));
        getter3.init(driver.setCurrentNodeForProcessorContext(table3.name));
        inputTopic.pipeInput("A", 1, 5L);
        inputTopic.pipeInput("B", 1, 10L);
        inputTopic.pipeInput("C", 1, 15L);
        assertNull(getter2.get("A"));
        assertNull(getter2.get("B"));
        assertNull(getter2.get("C"));
        assertEquals(ValueAndTimestamp.make(1, 5L), getter3.get("A"));
        assertEquals(ValueAndTimestamp.make(1, 10L), getter3.get("B"));
        assertEquals(ValueAndTimestamp.make(1, 15L), getter3.get("C"));
        inputTopic.pipeInput("A", 2, 10L);
        inputTopic.pipeInput("B", 2, 5L);
        assertEquals(ValueAndTimestamp.make(2, 10L), getter2.get("A"));
        assertEquals(ValueAndTimestamp.make(2, 5L), getter2.get("B"));
        assertNull(getter2.get("C"));
        assertNull(getter3.get("A"));
        assertNull(getter3.get("B"));
        assertEquals(ValueAndTimestamp.make(1, 15L), getter3.get("C"));
        inputTopic.pipeInput("A", 3, 15L);
        assertNull(getter2.get("A"));
        assertEquals(ValueAndTimestamp.make(2, 5L), getter2.get("B"));
        assertNull(getter2.get("C"));
        assertEquals(ValueAndTimestamp.make(3, 15L), getter3.get("A"));
        assertNull(getter3.get("B"));
        assertEquals(ValueAndTimestamp.make(1, 15L), getter3.get("C"));
        inputTopic.pipeInput("A", null, 10L);
        inputTopic.pipeInput("B", null, 20L);
        assertNull(getter2.get("A"));
        assertNull(getter2.get("B"));
        assertNull(getter2.get("C"));
        assertNull(getter3.get("A"));
        assertNull(getter3.get("B"));
        assertEquals(ValueAndTimestamp.make(1, 15L), getter3.get("C"));
    }
}

Example 93 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableKTableForeignKeyJoinScenarioTest method shouldUseExpectedTopicsWithSerde.

@Test
public void shouldUseExpectedTopicsWithSerde() {
    final String applicationId = "ktable-ktable-joinOnForeignKey";
    final Properties streamsConfig = mkProperties(mkMap(mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, applicationId), mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath())));
    final UniqueTopicSerdeScope serdeScope = new UniqueTopicSerdeScope();
    final StreamsBuilder builder = new StreamsBuilder();
    final KTable<Integer, String> left = builder.table(LEFT_TABLE, Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    final KTable<Integer, String> right = builder.table(RIGHT_TABLE, Consumed.with(serdeScope.decorateSerde(Serdes.Integer(), streamsConfig, true), serdeScope.decorateSerde(Serdes.String(), streamsConfig, false)));
    left.join(right, value -> Integer.parseInt(value.split("\\|")[1]), (value1, value2) -> "(" + value1 + "," + value2 + ")", Materialized.with(null, serdeScope.decorateSerde(Serdes.String(), streamsConfig, false))).toStream().to(OUTPUT);
    final Topology topology = builder.build(streamsConfig);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, streamsConfig)) {
        final TestInputTopic<Integer, String> leftInput = driver.createInputTopic(LEFT_TABLE, new IntegerSerializer(), new StringSerializer());
        final TestInputTopic<Integer, String> rightInput = driver.createInputTopic(RIGHT_TABLE, new IntegerSerializer(), new StringSerializer());
        leftInput.pipeInput(2, "lhsValue1|1");
        rightInput.pipeInput(1, "rhsValue1");
    }
    // verifying primarily that no extra pseudo-topics were used, but it's nice to also verify the rest of the
    // topics our serdes serialize data for
    assertThat(serdeScope.registeredTopics(), is(mkSet(// expected pseudo-topics
    applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-fk--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-pk--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic-vh--value", // internal topics
    applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-REGISTRATION-0000000006-topic--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--key", applicationId + "-KTABLE-FK-JOIN-SUBSCRIPTION-RESPONSE-0000000014-topic--value", applicationId + "-left_table-STATE-STORE-0000000000-changelog--key", applicationId + "-left_table-STATE-STORE-0000000000-changelog--value", applicationId + "-right_table-STATE-STORE-0000000003-changelog--key", applicationId + "-right_table-STATE-STORE-0000000003-changelog--value", // output topics
    "output-topic--key", "output-topic--value")));
}

Example 94 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KTableKTableLeftJoinTest method testNotSendingOldValue.

@Test
public void testNotSendingOldValue() {
    final StreamsBuilder builder = new StreamsBuilder();
    final int[] expectedKeys = new int[] { 0, 1, 2, 3 };
    final KTable<Integer, String> table1;
    final KTable<Integer, String> table2;
    final KTable<Integer, String> joined;
    final MockApiProcessorSupplier<Integer, String, Void, Void> supplier = new MockApiProcessorSupplier<>();
    table1 = builder.table(topic1, consumed);
    table2 = builder.table(topic2, consumed);
    joined = table1.leftJoin(table2, MockValueJoiner.TOSTRING_JOINER);
    final Topology topology = builder.build().addProcessor("proc", supplier, ((KTableImpl<?, ?, ?>) joined).name);
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<Integer, String> inputTopic1 = driver.createInputTopic(topic1, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final TestInputTopic<Integer, String> inputTopic2 = driver.createInputTopic(topic2, Serdes.Integer().serializer(), Serdes.String().serializer(), Instant.ofEpochMilli(0L), Duration.ZERO);
        final MockApiProcessor<Integer, String, Void, Void> proc = supplier.theCapturedProcessor();
        assertTrue(((KTableImpl<?, ?, ?>) table1).sendingOldValueEnabled());
        assertFalse(((KTableImpl<?, ?, ?>) table2).sendingOldValueEnabled());
        assertFalse(((KTableImpl<?, ?, ?>) joined).sendingOldValueEnabled());
        // push two items to the primary stream. the other table is empty
        for (int i = 0; i < 2; i++) {
            inputTopic1.pipeInput(expectedKeys[i], "X" + expectedKeys[i], 5L + i);
        }
        // pass tuple with null key, it will be discarded in join process
        inputTopic1.pipeInput(null, "SomeVal", 42L);
        // left: X0:0 (ts: 5), X1:1 (ts: 6)
        // right:
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+null", null), 5), new KeyValueTimestamp<>(1, new Change<>("X1+null", null), 6));
        // push two items to the other stream. this should produce two items.
        for (int i = 0; i < 2; i++) {
            inputTopic2.pipeInput(expectedKeys[i], "Y" + expectedKeys[i], 10L * i);
        }
        // pass tuple with null key, it will be discarded in join process
        inputTopic2.pipeInput(null, "AnotherVal", 73L);
        // left: X0:0 (ts: 5), X1:1 (ts: 6)
        // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("X0+Y0", null), 5), new KeyValueTimestamp<>(1, new Change<>("X1+Y1", null), 10));
        // push all four items to the primary stream. this should produce four items.
        for (final int expectedKey : expectedKeys) {
            inputTopic1.pipeInput(expectedKey, "XX" + expectedKey, 7L);
        }
        // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
        // right: Y0:0 (ts: 0), Y1:1 (ts: 10)
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XX0+Y0", null), 7), new KeyValueTimestamp<>(1, new Change<>("XX1+Y1", null), 10), new KeyValueTimestamp<>(2, new Change<>("XX2+null", null), 7), new KeyValueTimestamp<>(3, new Change<>("XX3+null", null), 7));
        // push all items to the other stream. this should produce four items.
        for (final int expectedKey : expectedKeys) {
            inputTopic2.pipeInput(expectedKey, "YY" + expectedKey, expectedKey * 5L);
        }
        // left: XX0:0 (ts: 7), XX1:1 (ts: 7), XX2:2 (ts: 7), XX3:3 (ts: 7)
        // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XX0+YY0", null), 7), new KeyValueTimestamp<>(1, new Change<>("XX1+YY1", null), 7), new KeyValueTimestamp<>(2, new Change<>("XX2+YY2", null), 10), new KeyValueTimestamp<>(3, new Change<>("XX3+YY3", null), 15));
        // push all four items to the primary stream. this should produce four items.
        for (final int expectedKey : expectedKeys) {
            inputTopic1.pipeInput(expectedKey, "XXX" + expectedKey, 6L);
        }
        // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
        // right: YY0:0 (ts: 0), YY1:1 (ts: 5), YY2:2 (ts: 10), YY3:3 (ts: 15)
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+YY0", null), 6), new KeyValueTimestamp<>(1, new Change<>("XXX1+YY1", null), 6), new KeyValueTimestamp<>(2, new Change<>("XXX2+YY2", null), 10), new KeyValueTimestamp<>(3, new Change<>("XXX3+YY3", null), 15));
        // push two items with null to the other stream as deletes. this should produce two item.
        inputTopic2.pipeInput(expectedKeys[0], null, 5L);
        inputTopic2.pipeInput(expectedKeys[1], null, 7L);
        // left: XXX0:0 (ts: 6), XXX1:1 (ts: 6), XXX2:2 (ts: 6), XXX3:3 (ts: 6)
        // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXX0+null", null), 6), new KeyValueTimestamp<>(1, new Change<>("XXX1+null", null), 7));
        // push all four items to the primary stream. this should produce four items.
        for (final int expectedKey : expectedKeys) {
            inputTopic1.pipeInput(expectedKey, "XXXX" + expectedKey, 13L);
        }
        // left: XXXX0:0 (ts: 13), XXXX1:1 (ts: 13), XXXX2:2 (ts: 13), XXXX3:3 (ts: 13)
        // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>("XXXX0+null", null), 13), new KeyValueTimestamp<>(1, new Change<>("XXXX1+null", null), 13), new KeyValueTimestamp<>(2, new Change<>("XXXX2+YY2", null), 13), new KeyValueTimestamp<>(3, new Change<>("XXXX3+YY3", null), 15));
        // push four items to the primary stream with null. this should produce four items.
        inputTopic1.pipeInput(expectedKeys[0], null, 0L);
        inputTopic1.pipeInput(expectedKeys[1], null, 42L);
        inputTopic1.pipeInput(expectedKeys[2], null, 5L);
        inputTopic1.pipeInput(expectedKeys[3], null, 20L);
        // left:
        // right: YY2:2 (ts: 10), YY3:3 (ts: 15)
        proc.checkAndClearProcessResult(new KeyValueTimestamp<>(0, new Change<>(null, null), 0), new KeyValueTimestamp<>(1, new Change<>(null, null), 42), new KeyValueTimestamp<>(2, new Change<>(null, null), 10), new KeyValueTimestamp<>(3, new Change<>(null, null), 20));
    }
}

Example 95 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KStreamSplitTest method withDriver.

private void withDriver(final Consumer<TopologyTestDriver> test) {
    final int[] expectedKeys = new int[] { -1, 0, 1, 2, 3, 4, 5, 6, 7 };
    final Topology topology = builder.build();
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<Integer, String> inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer());
        for (final int expectedKey : expectedKeys) {
            inputTopic.pipeInput(expectedKey, "V" + expectedKey);
        }
        test.accept(driver);
    }
}