Search in sources :

Example 1 with Pseudograph

use of org.jgrapht.graph.Pseudograph in project storm by nathanmarz.

the class TridentTopology method getGroupParallelisms.

private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) {
    UndirectedGraph<Group, Object> equivs = new Pseudograph<Group, Object>(Object.class);
    for (Group g : groups) {
        equivs.addVertex(g);
    }
    for (Group g : groups) {
        for (PartitionNode n : externalGroupInputs(g)) {
            if (isIdentityPartition(n)) {
                Node parent = TridentUtils.getParent(graph, n);
                Group parentGroup = grouper.nodeGroup(parent);
                if (parentGroup != null && !parentGroup.equals(g)) {
                    equivs.addEdge(parentGroup, g);
                }
            }
        }
    }
    Map<Group, Integer> ret = new HashMap();
    List<Set<Group>> equivGroups = new ConnectivityInspector<Group, Object>(equivs).connectedSets();
    for (Set<Group> equivGroup : equivGroups) {
        Integer fixedP = getFixedParallelism(equivGroup);
        Integer maxP = getMaxParallelism(equivGroup);
        if (fixedP != null && maxP != null && maxP < fixedP) {
            throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP);
        }
        Integer p = 1;
        for (Group g : equivGroup) {
            for (Node n : g.nodes) {
                if (n.parallelismHint != null) {
                    p = Math.max(p, n.parallelismHint);
                }
            }
        }
        if (maxP != null)
            p = Math.min(maxP, p);
        if (fixedP != null)
            p = fixedP;
        for (Group g : equivGroup) {
            ret.put(g, p);
        }
    }
    return ret;
}
Also used : Group(storm.trident.graph.Group) Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) SpoutNode(storm.trident.planner.SpoutNode) ProcessorNode(storm.trident.planner.ProcessorNode) PartitionNode(storm.trident.planner.PartitionNode) Node(storm.trident.planner.Node) Pseudograph(org.jgrapht.graph.Pseudograph) PartitionNode(storm.trident.planner.PartitionNode)

Example 2 with Pseudograph

use of org.jgrapht.graph.Pseudograph in project storm by apache.

the class TridentTopology method getGroupParallelisms.

private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) {
    UndirectedGraph<Group, Object> equivs = new Pseudograph<>(Object.class);
    for (Group g : groups) {
        equivs.addVertex(g);
    }
    for (Group g : groups) {
        for (PartitionNode n : externalGroupInputs(g)) {
            if (isIdentityPartition(n)) {
                Node parent = TridentUtils.getParent(graph, n);
                Group parentGroup = grouper.nodeGroup(parent);
                if (parentGroup != null && !parentGroup.equals(g)) {
                    equivs.addEdge(parentGroup, g);
                }
            }
        }
    }
    Map<Group, Integer> ret = new HashMap<>();
    List<Set<Group>> equivGroups = new ConnectivityInspector<>(equivs).connectedSets();
    for (Set<Group> equivGroup : equivGroups) {
        Integer fixedP = getFixedParallelism(equivGroup);
        Integer maxP = getMaxParallelism(equivGroup);
        if (fixedP != null && maxP != null && maxP < fixedP) {
            throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP);
        }
        Integer p = 1;
        for (Group g : equivGroup) {
            for (Node n : g.nodes) {
                if (n.parallelismHint != null) {
                    p = Math.max(p, n.parallelismHint);
                }
            }
        }
        if (maxP != null)
            p = Math.min(maxP, p);
        if (fixedP != null)
            p = fixedP;
        for (Group g : equivGroup) {
            ret.put(g, p);
        }
    }
    return ret;
}
Also used : Group(org.apache.storm.trident.graph.Group) ProcessorNode(org.apache.storm.trident.planner.ProcessorNode) Node(org.apache.storm.trident.planner.Node) PartitionNode(org.apache.storm.trident.planner.PartitionNode) SpoutNode(org.apache.storm.trident.planner.SpoutNode) Pseudograph(org.jgrapht.graph.Pseudograph) PartitionNode(org.apache.storm.trident.planner.PartitionNode)

Example 3 with Pseudograph

use of org.jgrapht.graph.Pseudograph in project jstorm by alibaba.

the class TridentTopology method getGroupParallelisms.

private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) {
    UndirectedGraph<Group, Object> equivs = new Pseudograph<>(Object.class);
    for (Group g : groups) {
        equivs.addVertex(g);
    }
    for (Group g : groups) {
        for (PartitionNode n : externalGroupInputs(g)) {
            if (isIdentityPartition(n)) {
                Node parent = TridentUtils.getParent(graph, n);
                Group parentGroup = grouper.nodeGroup(parent);
                if (parentGroup != null && !parentGroup.equals(g)) {
                    equivs.addEdge(parentGroup, g);
                }
            }
        }
    }
    Map<Group, Integer> ret = new HashMap<>();
    List<Set<Group>> equivGroups = new ConnectivityInspector<>(equivs).connectedSets();
    for (Set<Group> equivGroup : equivGroups) {
        Integer fixedP = getFixedParallelism(equivGroup);
        Integer maxP = getMaxParallelism(equivGroup);
        if (fixedP != null && maxP != null && maxP < fixedP) {
            throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP);
        }
        Integer p = 1;
        for (Group g : equivGroup) {
            for (Node n : g.nodes) {
                if (n.parallelismHint != null) {
                    p = Math.max(p, n.parallelismHint);
                }
            }
        }
        if (maxP != null)
            p = Math.min(maxP, p);
        if (fixedP != null)
            p = fixedP;
        for (Group g : equivGroup) {
            ret.put(g, p);
        }
    }
    return ret;
}
Also used : Group(storm.trident.graph.Group) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) SpoutNode(storm.trident.planner.SpoutNode) ProcessorNode(storm.trident.planner.ProcessorNode) PartitionNode(storm.trident.planner.PartitionNode) Node(storm.trident.planner.Node) Pseudograph(org.jgrapht.graph.Pseudograph) PartitionNode(storm.trident.planner.PartitionNode)

Aggregations

Pseudograph (org.jgrapht.graph.Pseudograph)3 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 Set (java.util.Set)2 Group (storm.trident.graph.Group)2 Node (storm.trident.planner.Node)2 PartitionNode (storm.trident.planner.PartitionNode)2 ProcessorNode (storm.trident.planner.ProcessorNode)2 SpoutNode (storm.trident.planner.SpoutNode)2 LinkedHashMap (java.util.LinkedHashMap)1 LinkedHashSet (java.util.LinkedHashSet)1 Group (org.apache.storm.trident.graph.Group)1 Node (org.apache.storm.trident.planner.Node)1 PartitionNode (org.apache.storm.trident.planner.PartitionNode)1 ProcessorNode (org.apache.storm.trident.planner.ProcessorNode)1 SpoutNode (org.apache.storm.trident.planner.SpoutNode)1