Examples with Topology org.apache.kafka.streams.Topology used on opensource projects

Example 1 with Topology

use of org.apache.kafka.streams.Topology in project ksql by confluentinc.

the class JoinNodeTest method shouldHaveLeftJoin.

@Test
public void shouldHaveLeftJoin() {
    setupTopicClientExpectations(1, 1);
    buildJoin();
    final Topology topology = builder.build();
    final TopologyDescription.Processor leftJoin = (TopologyDescription.Processor) getNodeByName(topology, "KSTREAM-LEFTJOIN-0000000014");
    final List<String> predecessors = leftJoin.predecessors().stream().map(TopologyDescription.Node::name).collect(Collectors.toList());
    assertThat(leftJoin.stores(), equalTo(Utils.mkSet("KSTREAM-REDUCE-STATE-STORE-0000000003")));
    assertThat(predecessors, equalTo(Collections.singletonList("KSTREAM-SOURCE-0000000013")));
}

Example 2 with Topology

use of org.apache.kafka.streams.Topology in project ksql by confluentinc.

the class PlanTestUtil method getNodeByName.

static TopologyDescription.Node getNodeByName(final Topology topology, final String nodeName) {
    final TopologyDescription description = topology.describe();
    final Set<TopologyDescription.Subtopology> subtopologies = description.subtopologies();
    List<TopologyDescription.Node> nodes = subtopologies.stream().flatMap(subtopology -> subtopology.nodes().stream()).collect(Collectors.toList());
    final Map<String, List<TopologyDescription.Node>> nodesByName = nodes.stream().collect(Collectors.groupingBy(TopologyDescription.Node::name));
    return nodesByName.get(nodeName).get(0);
}

Example 3 with Topology

use of org.apache.kafka.streams.Topology in project apache-kafka-on-k8s by banzaicloud.

the class RestoreIntegrationTest method shouldProcessDataFromStoresWithLoggingDisabled.

@Test
public void shouldProcessDataFromStoresWithLoggingDisabled() throws InterruptedException, ExecutionException {
    IntegrationTestUtils.produceKeyValuesSynchronously(INPUT_STREAM_2, Arrays.asList(KeyValue.pair(1, 1), KeyValue.pair(2, 2), KeyValue.pair(3, 3)), TestUtils.producerConfig(CLUSTER.bootstrapServers(), IntegerSerializer.class, IntegerSerializer.class), CLUSTER.time);
    final KeyValueBytesStoreSupplier lruMapSupplier = Stores.lruMap(INPUT_STREAM_2, 10);
    final StoreBuilder<KeyValueStore<Integer, Integer>> storeBuilder = new KeyValueStoreBuilder<>(lruMapSupplier, Serdes.Integer(), Serdes.Integer(), CLUSTER.time).withLoggingDisabled();
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    streamsBuilder.addStateStore(storeBuilder);
    final KStream<Integer, Integer> stream = streamsBuilder.stream(INPUT_STREAM_2);
    final CountDownLatch processorLatch = new CountDownLatch(3);
    stream.process(new ProcessorSupplier<Integer, Integer>() {

        @Override
        public Processor<Integer, Integer> get() {
            return new KeyValueStoreProcessor(INPUT_STREAM_2, processorLatch);
        }
    }, INPUT_STREAM_2);
    final Topology topology = streamsBuilder.build();
    kafkaStreams = new KafkaStreams(topology, props(applicationId + "-logging-disabled"));
    final CountDownLatch latch = new CountDownLatch(1);
    kafkaStreams.setStateListener(new KafkaStreams.StateListener() {

        @Override
        public void onChange(final KafkaStreams.State newState, final KafkaStreams.State oldState) {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                latch.countDown();
            }
        }
    });
    kafkaStreams.start();
    latch.await(30, TimeUnit.SECONDS);
    assertTrue(processorLatch.await(30, TimeUnit.SECONDS));
}

Example 4 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KStreamKStreamJoinTest method shouldCreateRepartitionTopicsWithUserProvidedName.

@Test
public void shouldCreateRepartitionTopicsWithUserProvidedName() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Properties props = new Properties();
    props.put(StreamsConfig.TOPOLOGY_OPTIMIZATION_CONFIG, StreamsConfig.NO_OPTIMIZATION);
    final KStream<String, String> stream1 = builder.stream("topic", Consumed.with(Serdes.String(), Serdes.String()));
    final KStream<String, String> stream2 = builder.stream("topic2", Consumed.with(Serdes.String(), Serdes.String()));
    final KStream<String, String> stream3 = builder.stream("topic3", Consumed.with(Serdes.String(), Serdes.String()));
    final KStream<String, String> newStream = stream1.map((k, v) -> new KeyValue<>(v, k));
    final StreamJoined<String, String, String> streamJoined = StreamJoined.with(Serdes.String(), Serdes.String(), Serdes.String());
    newStream.join(stream2, (value1, value2) -> value1 + value2, JoinWindows.ofTimeDifferenceWithNoGrace(ofMillis(100)), streamJoined.withName("first-join")).to("out-one");
    newStream.join(stream3, (value1, value2) -> value1 + value2, JoinWindows.ofTimeDifferenceWithNoGrace(ofMillis(100)), streamJoined.withName("second-join")).to("out-two");
    final Topology topology = builder.build(props);
    System.out.println(topology.describe().toString());
    assertEquals(expectedTopologyWithUserNamedRepartitionTopics, topology.describe().toString());
}

Example 5 with Topology

use of org.apache.kafka.streams.Topology in project kafka by apache.

the class KStreamImplTest method shouldSupportStreamTableJoinWithKStreamToKTable.

@Test
public void shouldSupportStreamTableJoinWithKStreamToKTable() {
    final StreamsBuilder builder = new StreamsBuilder();
    final Consumed<String, String> consumed = Consumed.with(Serdes.String(), Serdes.String());
    final String streamTopic = "streamTopic";
    final String tableTopic = "tableTopic";
    final String outputTopic = "output";
    final KStream<String, String> stream = builder.stream(streamTopic, consumed);
    final KTable<String, String> table = builder.stream(tableTopic, consumed).toTable();
    stream.join(table, MockValueJoiner.TOSTRING_JOINER).to(outputTopic);
    final Topology topology = builder.build(props);
    final String topologyDescription = topology.describe().toString();
    assertThat(topologyDescription, equalTo("Topologies:\n" + "   Sub-topology: 0\n" + "    Source: KSTREAM-SOURCE-0000000000 (topics: [streamTopic])\n" + "      --> KSTREAM-JOIN-0000000004\n" + "    Processor: KSTREAM-JOIN-0000000004 (stores: [KSTREAM-TOTABLE-STATE-STORE-0000000003])\n" + "      --> KSTREAM-SINK-0000000005\n" + "      <-- KSTREAM-SOURCE-0000000000\n" + "    Source: KSTREAM-SOURCE-0000000001 (topics: [tableTopic])\n" + "      --> KSTREAM-TOTABLE-0000000002\n" + "    Sink: KSTREAM-SINK-0000000005 (topic: output)\n" + "      <-- KSTREAM-JOIN-0000000004\n" + "    Processor: KSTREAM-TOTABLE-0000000002 (stores: [KSTREAM-TOTABLE-STATE-STORE-0000000003])\n" + "      --> none\n" + "      <-- KSTREAM-SOURCE-0000000001\n\n"));
    try (final TopologyTestDriver driver = new TopologyTestDriver(topology, props)) {
        final TestInputTopic<String, String> left = driver.createInputTopic(streamTopic, new StringSerializer(), new StringSerializer());
        final TestInputTopic<String, String> right = driver.createInputTopic(tableTopic, new StringSerializer(), new StringSerializer());
        final TestOutputTopic<String, String> output = driver.createOutputTopic(outputTopic, new StringDeserializer(), new StringDeserializer());
        right.pipeInput("lhs1", "rhsValue1");
        right.pipeInput("rhs2", "rhsValue2");
        right.pipeInput("lhs3", "rhsValue3");
        assertThat(output.readKeyValuesToMap(), is(emptyMap()));
        left.pipeInput("lhs1", "lhsValue1");
        left.pipeInput("lhs2", "lhsValue2");
        final Map<String, String> expected = mkMap(mkEntry("lhs1", "lhsValue1+rhsValue1"));
        assertThat(output.readKeyValuesToMap(), is(expected));
        left.pipeInput("lhs3", "lhsValue3");
        assertThat(output.readKeyValuesToMap(), is(mkMap(mkEntry("lhs3", "lhsValue3+rhsValue3"))));
        left.pipeInput("lhs1", "lhsValue4");
        assertThat(output.readKeyValuesToMap(), is(mkMap(mkEntry("lhs1", "lhsValue4+rhsValue1"))));
    }
}