Search in sources :

Example 6 with PartitionNode

use of storm.trident.planner.PartitionNode in project jstorm by alibaba.

the class TridentTopology method build.

public StormTopology build() {
    // Transaction is not compatible with jstorm batch mode(task.batch.tuple)
    // so we close batch mode via system property
    System.setProperty(ConfigExtension.TASK_BATCH_TUPLE, "false");
    DefaultDirectedGraph<Node, IndexedEdge> graph = (DefaultDirectedGraph) _graph.clone();
    completeDRPC(graph, _colocate, _gen);
    List<SpoutNode> spoutNodes = new ArrayList<>();
    // can be regular nodes (static state) or processor nodes
    Set<Node> boltNodes = new LinkedHashSet<>();
    for (Node n : graph.vertexSet()) {
        if (n instanceof SpoutNode) {
            spoutNodes.add((SpoutNode) n);
        } else if (!(n instanceof PartitionNode)) {
            boltNodes.add(n);
        }
    }
    Set<Group> initialGroups = new LinkedHashSet<>();
    for (List<Node> colocate : _colocate.values()) {
        Group g = new Group(graph, colocate);
        boltNodes.removeAll(colocate);
        initialGroups.add(g);
    }
    for (Node n : boltNodes) {
        initialGroups.add(new Group(graph, n));
    }
    GraphGrouper grouper = new GraphGrouper(graph, initialGroups);
    grouper.mergeFully();
    Collection<Group> mergedGroups = grouper.getAllGroups();
    // add identity partitions between groups
    for (IndexedEdge<Node> e : new HashSet<>(graph.edgeSet())) {
        if (!(e.source instanceof PartitionNode) && !(e.target instanceof PartitionNode)) {
            Group g1 = grouper.nodeGroup(e.source);
            Group g2 = grouper.nodeGroup(e.target);
            // g1 being null means the source is a spout node
            if (g1 == null && !(e.source instanceof SpoutNode))
                throw new RuntimeException("Planner exception: Null source group must indicate a spout node at this phase of planning");
            if (g1 == null || !g1.equals(g2)) {
                graph.removeEdge(e);
                PartitionNode pNode = makeIdentityPartition(e.source);
                graph.addVertex(pNode);
                graph.addEdge(e.source, pNode, new IndexedEdge(e.source, pNode, 0));
                graph.addEdge(pNode, e.target, new IndexedEdge(pNode, e.target, e.index));
            }
        }
    }
    // if one group subscribes to the same stream with same partitioning multiple times,
    // merge those together (otherwise can end up with many output streams created for that partitioning
    // if need to split into multiple output streams because of same input having different
    // partitioning to the group)
    // this is because can't currently merge splitting logic into a spout
    // not the most kosher algorithm here, since the grouper indexes are being trounced via the adding of nodes to random groups, but it 
    // works out
    List<Node> forNewGroups = new ArrayList<>();
    for (Group g : mergedGroups) {
        for (PartitionNode n : extraPartitionInputs(g)) {
            Node idNode = makeIdentityNode(n.allOutputFields);
            Node newPartitionNode = new PartitionNode(idNode.streamId, n.name, idNode.allOutputFields, n.thriftGrouping);
            Node parentNode = TridentUtils.getParent(graph, n);
            Set<IndexedEdge> outgoing = graph.outgoingEdgesOf(n);
            graph.removeVertex(n);
            graph.addVertex(idNode);
            graph.addVertex(newPartitionNode);
            addEdge(graph, parentNode, idNode, 0);
            addEdge(graph, idNode, newPartitionNode, 0);
            for (IndexedEdge e : outgoing) {
                addEdge(graph, newPartitionNode, e.target, e.index);
            }
            Group parentGroup = grouper.nodeGroup(parentNode);
            if (parentGroup == null) {
                forNewGroups.add(idNode);
            } else {
                parentGroup.nodes.add(idNode);
            }
        }
    }
    for (Node n : forNewGroups) {
        grouper.addGroup(new Group(graph, n));
    }
    // add in spouts as groups so we can get parallelisms
    for (Node n : spoutNodes) {
        grouper.addGroup(new Group(graph, n));
    }
    grouper.reindex();
    mergedGroups = grouper.getAllGroups();
    Map<Node, String> batchGroupMap = new HashMap<>();
    List<Set<Node>> connectedComponents = new ConnectivityInspector<>(graph).connectedSets();
    for (int i = 0; i < connectedComponents.size(); i++) {
        String groupId = "bg" + i;
        for (Node n : connectedComponents.get(i)) {
            batchGroupMap.put(n, groupId);
        }
    }
    //        System.out.println("GRAPH:");
    //        System.out.println(graph);
    Map<Group, Integer> parallelisms = getGroupParallelisms(graph, grouper, mergedGroups);
    TridentTopologyBuilder builder = new TridentTopologyBuilder();
    Map<Node, String> spoutIds = genSpoutIds(spoutNodes);
    Map<Group, String> boltIds = genBoltIds(mergedGroups);
    Map defaults = Utils.readDefaultConfig();
    for (SpoutNode sn : spoutNodes) {
        Integer parallelism = parallelisms.get(grouper.nodeGroup(sn));
        Map<String, Number> spoutRes = null;
        spoutRes = mergeDefaultResources(sn.getResources(), defaults);
        Number onHeap = spoutRes.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
        Number offHeap = spoutRes.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB);
        Number cpuLoad = spoutRes.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT);
        if (sn.type == SpoutNode.SpoutType.DRPC) {
            builder.setBatchPerTupleSpout(spoutIds.get(sn), sn.streamId, (IRichSpout) sn.spout, parallelism, batchGroupMap.get(sn)).setMemoryLoad(onHeap, offHeap).setCPULoad(cpuLoad);
        } else {
            ITridentSpout s;
            if (sn.spout instanceof IBatchSpout) {
                s = new BatchSpoutExecutor((IBatchSpout) sn.spout);
            } else if (sn.spout instanceof ITridentSpout) {
                s = (ITridentSpout) sn.spout;
            } else {
                throw new RuntimeException("Regular rich spouts not supported yet... try wrapping in a RichSpoutBatchExecutor");
            // TODO: handle regular rich spout without batches (need lots of updates to support this throughout)
            }
            builder.setSpout(spoutIds.get(sn), sn.streamId, sn.txId, s, parallelism, batchGroupMap.get(sn)).setMemoryLoad(onHeap, offHeap).setCPULoad(cpuLoad);
        }
    }
    for (Group g : mergedGroups) {
        if (!isSpoutGroup(g)) {
            Integer p = parallelisms.get(g);
            Map<String, String> streamToGroup = getOutputStreamBatchGroups(g, batchGroupMap);
            Map<String, Number> groupRes = mergeDefaultResources(g.getResources(), defaults);
            Number onHeap = groupRes.get(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB);
            Number offHeap = groupRes.get(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB);
            Number cpuLoad = groupRes.get(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT);
            BoltDeclarer d = builder.setBolt(boltIds.get(g), new SubtopologyBolt(graph, g.nodes, batchGroupMap), p, committerBatches(g, batchGroupMap), streamToGroup).setMemoryLoad(onHeap, offHeap).setCPULoad(cpuLoad);
            Collection<PartitionNode> inputs = uniquedSubscriptions(externalGroupInputs(g));
            for (PartitionNode n : inputs) {
                Node parent = TridentUtils.getParent(graph, n);
                String componentId = parent instanceof SpoutNode ? spoutIds.get(parent) : boltIds.get(grouper.nodeGroup(parent));
                d.grouping(new GlobalStreamId(componentId, n.streamId), n.thriftGrouping);
            }
        }
    }
    return builder.buildTopology();
}
Also used : LinkedHashSet(java.util.LinkedHashSet) Group(storm.trident.graph.Group) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) IBatchSpout(storm.trident.spout.IBatchSpout) DefaultDirectedGraph(org.jgrapht.graph.DefaultDirectedGraph) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) SpoutNode(storm.trident.planner.SpoutNode) ProcessorNode(storm.trident.planner.ProcessorNode) PartitionNode(storm.trident.planner.PartitionNode) Node(storm.trident.planner.Node) ArrayList(java.util.ArrayList) GraphGrouper(storm.trident.graph.GraphGrouper) IndexedEdge(storm.trident.util.IndexedEdge) BatchSpoutExecutor(storm.trident.spout.BatchSpoutExecutor) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) TridentTopologyBuilder(storm.trident.topology.TridentTopologyBuilder) SpoutNode(storm.trident.planner.SpoutNode) PartitionNode(storm.trident.planner.PartitionNode) BoltDeclarer(backtype.storm.topology.BoltDeclarer) GlobalStreamId(backtype.storm.generated.GlobalStreamId) SubtopologyBolt(storm.trident.planner.SubtopologyBolt) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) TreeMap(java.util.TreeMap) ITridentSpout(storm.trident.spout.ITridentSpout)

Example 7 with PartitionNode

use of storm.trident.planner.PartitionNode in project jstorm by alibaba.

the class TridentTopology method getGroupParallelisms.

private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) {
    UndirectedGraph<Group, Object> equivs = new Pseudograph<>(Object.class);
    for (Group g : groups) {
        equivs.addVertex(g);
    }
    for (Group g : groups) {
        for (PartitionNode n : externalGroupInputs(g)) {
            if (isIdentityPartition(n)) {
                Node parent = TridentUtils.getParent(graph, n);
                Group parentGroup = grouper.nodeGroup(parent);
                if (parentGroup != null && !parentGroup.equals(g)) {
                    equivs.addEdge(parentGroup, g);
                }
            }
        }
    }
    Map<Group, Integer> ret = new HashMap<>();
    List<Set<Group>> equivGroups = new ConnectivityInspector<>(equivs).connectedSets();
    for (Set<Group> equivGroup : equivGroups) {
        Integer fixedP = getFixedParallelism(equivGroup);
        Integer maxP = getMaxParallelism(equivGroup);
        if (fixedP != null && maxP != null && maxP < fixedP) {
            throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP);
        }
        Integer p = 1;
        for (Group g : equivGroup) {
            for (Node n : g.nodes) {
                if (n.parallelismHint != null) {
                    p = Math.max(p, n.parallelismHint);
                }
            }
        }
        if (maxP != null)
            p = Math.min(maxP, p);
        if (fixedP != null)
            p = fixedP;
        for (Group g : equivGroup) {
            ret.put(g, p);
        }
    }
    return ret;
}
Also used : Group(storm.trident.graph.Group) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) SpoutNode(storm.trident.planner.SpoutNode) ProcessorNode(storm.trident.planner.ProcessorNode) PartitionNode(storm.trident.planner.PartitionNode) Node(storm.trident.planner.Node) Pseudograph(org.jgrapht.graph.Pseudograph) PartitionNode(storm.trident.planner.PartitionNode)

Example 8 with PartitionNode

use of storm.trident.planner.PartitionNode in project jstorm by alibaba.

the class TridentTopology method getOutputStreamBatchGroups.

private static Map<String, String> getOutputStreamBatchGroups(Group g, Map<Node, String> batchGroupMap) {
    Map<String, String> ret = new HashMap<>();
    Set<PartitionNode> externalGroupOutputs = externalGroupOutputs(g);
    for (PartitionNode n : externalGroupOutputs) {
        ret.put(n.streamId, batchGroupMap.get(n));
    }
    return ret;
}
Also used : HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) PartitionNode(storm.trident.planner.PartitionNode)

Example 9 with PartitionNode

use of storm.trident.planner.PartitionNode in project storm by nathanmarz.

the class TridentTopology method extraPartitionInputs.

private static List<PartitionNode> extraPartitionInputs(Group g) {
    List<PartitionNode> ret = new ArrayList();
    Set<PartitionNode> inputs = externalGroupInputs(g);
    Map<String, List<PartitionNode>> grouped = new HashMap();
    for (PartitionNode n : inputs) {
        if (!grouped.containsKey(n.streamId)) {
            grouped.put(n.streamId, new ArrayList());
        }
        grouped.get(n.streamId).add(n);
    }
    for (List<PartitionNode> group : grouped.values()) {
        PartitionNode anchor = group.get(0);
        for (int i = 1; i < group.size(); i++) {
            PartitionNode n = group.get(i);
            if (!n.thriftGrouping.equals(anchor.thriftGrouping)) {
                ret.add(n);
            }
        }
    }
    return ret;
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) PartitionNode(storm.trident.planner.PartitionNode) List(java.util.List) ArrayList(java.util.ArrayList)

Example 10 with PartitionNode

use of storm.trident.planner.PartitionNode in project storm by nathanmarz.

the class TridentTopology method uniquedSubscriptions.

private static Collection<PartitionNode> uniquedSubscriptions(Set<PartitionNode> subscriptions) {
    Map<String, PartitionNode> ret = new HashMap();
    for (PartitionNode n : subscriptions) {
        PartitionNode curr = ret.get(n.streamId);
        if (curr != null && !curr.thriftGrouping.equals(n.thriftGrouping)) {
            throw new RuntimeException("Multiple subscriptions to the same stream with different groupings. Should be impossible since that is explicitly guarded against.");
        }
        ret.put(n.streamId, n);
    }
    return ret.values();
}
Also used : HashMap(java.util.HashMap) PartitionNode(storm.trident.planner.PartitionNode)

Aggregations

HashMap (java.util.HashMap)10 PartitionNode (storm.trident.planner.PartitionNode)10 LinkedHashMap (java.util.LinkedHashMap)5 ArrayList (java.util.ArrayList)4 HashSet (java.util.HashSet)4 Set (java.util.Set)4 Group (storm.trident.graph.Group)4 Node (storm.trident.planner.Node)4 ProcessorNode (storm.trident.planner.ProcessorNode)4 SpoutNode (storm.trident.planner.SpoutNode)4 GlobalStreamId (backtype.storm.generated.GlobalStreamId)2 BoltDeclarer (backtype.storm.topology.BoltDeclarer)2 LinkedHashSet (java.util.LinkedHashSet)2 List (java.util.List)2 DefaultDirectedGraph (org.jgrapht.graph.DefaultDirectedGraph)2 Pseudograph (org.jgrapht.graph.Pseudograph)2 GraphGrouper (storm.trident.graph.GraphGrouper)2 SubtopologyBolt (storm.trident.planner.SubtopologyBolt)2 BatchSpoutExecutor (storm.trident.spout.BatchSpoutExecutor)2 IBatchSpout (storm.trident.spout.IBatchSpout)2