Examples with ProcessorTopology org.apache.kafka.streams.processor.internals.ProcessorTopology used on opensource projects

Example 1 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class TopologyBuilder method build.

private ProcessorTopology build(Set<String> nodeGroup) {
    List<ProcessorNode> processorNodes = new ArrayList<>(nodeFactories.size());
    Map<String, ProcessorNode> processorMap = new HashMap<>();
    Map<String, SourceNode> topicSourceMap = new HashMap<>();
    Map<String, SinkNode> topicSinkMap = new HashMap<>();
    Map<String, StateStore> stateStoreMap = new LinkedHashMap<>();
    // create processor nodes in a topological order ("nodeFactories" is already topologically sorted)
    for (NodeFactory factory : nodeFactories.values()) {
        if (nodeGroup == null || nodeGroup.contains(factory.name)) {
            final ProcessorNode node = factory.build();
            processorNodes.add(node);
            processorMap.put(node.name(), node);
            if (factory instanceof ProcessorNodeFactory) {
                for (String parent : ((ProcessorNodeFactory) factory).parents) {
                    ProcessorNode<?, ?> parentNode = processorMap.get(parent);
                    parentNode.addChild(node);
                }
                for (String stateStoreName : ((ProcessorNodeFactory) factory).stateStoreNames) {
                    if (!stateStoreMap.containsKey(stateStoreName)) {
                        StateStore stateStore;
                        if (stateFactories.containsKey(stateStoreName)) {
                            final StateStoreSupplier supplier = stateFactories.get(stateStoreName).supplier;
                            stateStore = supplier.get();
                            // remember the changelog topic if this state store is change-logging enabled
                            if (supplier.loggingEnabled() && !storeToChangelogTopic.containsKey(stateStoreName)) {
                                final String changelogTopic = ProcessorStateManager.storeChangelogTopic(this.applicationId, stateStoreName);
                                storeToChangelogTopic.put(stateStoreName, changelogTopic);
                            }
                        } else {
                            stateStore = globalStateStores.get(stateStoreName);
                        }
                        stateStoreMap.put(stateStoreName, stateStore);
                    }
                }
            } else if (factory instanceof SourceNodeFactory) {
                final SourceNodeFactory sourceNodeFactory = (SourceNodeFactory) factory;
                final List<String> topics = (sourceNodeFactory.pattern != null) ? sourceNodeFactory.getTopics(subscriptionUpdates.getUpdates()) : sourceNodeFactory.topics;
                for (String topic : topics) {
                    if (internalTopicNames.contains(topic)) {
                        // prefix the internal topic name with the application id
                        topicSourceMap.put(decorateTopic(topic), (SourceNode) node);
                    } else {
                        topicSourceMap.put(topic, (SourceNode) node);
                    }
                }
            } else if (factory instanceof SinkNodeFactory) {
                final SinkNodeFactory sinkNodeFactory = (SinkNodeFactory) factory;
                for (String parent : sinkNodeFactory.parents) {
                    processorMap.get(parent).addChild(node);
                    if (internalTopicNames.contains(sinkNodeFactory.topic)) {
                        // prefix the internal topic name with the application id
                        topicSinkMap.put(decorateTopic(sinkNodeFactory.topic), (SinkNode) node);
                    } else {
                        topicSinkMap.put(sinkNodeFactory.topic, (SinkNode) node);
                    }
                }
            } else {
                throw new TopologyBuilderException("Unknown definition class: " + factory.getClass().getName());
            }
        }
    }
    return new ProcessorTopology(processorNodes, topicSourceMap, topicSinkMap, new ArrayList<>(stateStoreMap.values()), storeToChangelogTopic, new ArrayList<>(globalStateStores.values()));
}

Example 2 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class KStreamBuilderTest method shouldAddGlobalTablesToEachGroup.

@Test
public void shouldAddGlobalTablesToEachGroup() throws Exception {
    final String one = "globalTable";
    final String two = "globalTable2";
    final GlobalKTable<String, String> globalTable = builder.globalTable("table", one);
    final GlobalKTable<String, String> globalTable2 = builder.globalTable("table2", two);
    builder.table("not-global", "not-global");
    final KeyValueMapper<String, String, String> kvMapper = new KeyValueMapper<String, String, String>() {

        @Override
        public String apply(final String key, final String value) {
            return value;
        }
    };
    final KStream<String, String> stream = builder.stream("t1");
    stream.leftJoin(globalTable, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final KStream<String, String> stream2 = builder.stream("t2");
    stream2.leftJoin(globalTable2, kvMapper, MockValueJoiner.TOSTRING_JOINER);
    final Map<Integer, Set<String>> nodeGroups = builder.nodeGroups();
    for (Integer groupId : nodeGroups.keySet()) {
        final ProcessorTopology topology = builder.build(groupId);
        final List<StateStore> stateStores = topology.globalStateStores();
        final Set<String> names = new HashSet<>();
        for (StateStore stateStore : stateStores) {
            names.add(stateStore.name());
        }
        assertEquals(2, stateStores.size());
        assertTrue(names.contains(one));
        assertTrue(names.contains(two));
    }
}

Example 3 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class KStreamBuilderTest method shouldBuildGlobalTopologyWithAllGlobalTables.

@Test
public void shouldBuildGlobalTopologyWithAllGlobalTables() throws Exception {
    builder.globalTable("table", "globalTable");
    builder.globalTable("table2", "globalTable2");
    final ProcessorTopology topology = builder.buildGlobalStateTopology();
    final List<StateStore> stateStores = topology.globalStateStores();
    final Set<String> sourceTopics = topology.sourceTopics();
    assertEquals(Utils.mkSet("table", "table2"), sourceTopics);
    assertEquals(2, stateStores.size());
}

Example 4 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class KStreamBuilderTest method shouldNotMaterializeSourceKTableIfStateNameNotSpecified.

@Test
public void shouldNotMaterializeSourceKTableIfStateNameNotSpecified() throws Exception {
    builder.table("topic1", "table1");
    builder.table("topic2", null);
    final ProcessorTopology topology = builder.build(null);
    assertEquals(1, topology.stateStores().size());
    assertEquals("table1", topology.stateStores().get(0).name());
    assertEquals(1, topology.storeToChangelogTopic().size());
    assertEquals("topic1", topology.storeToChangelogTopic().get("table1"));
}

Example 5 with ProcessorTopology

use of org.apache.kafka.streams.processor.internals.ProcessorTopology in project kafka by apache.

the class KStreamBuilderTest method shouldBuildSimpleGlobalTableTopology.

@Test
public void shouldBuildSimpleGlobalTableTopology() throws Exception {
    builder.globalTable("table", "globalTable");
    final ProcessorTopology topology = builder.buildGlobalStateTopology();
    final List<StateStore> stateStores = topology.globalStateStores();
    assertEquals(1, stateStores.size());
    assertEquals("globalTable", stateStores.get(0).name());
}