Examples with Group storm.trident.graph.Group used on opensource projects

Example 6 with Group

use of storm.trident.graph.Group in project jstorm by alibaba.

the class TridentTopology method getGroupParallelisms.

private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) {
    UndirectedGraph<Group, Object> equivs = new Pseudograph<>(Object.class);
    for (Group g : groups) {
        equivs.addVertex(g);
    }
    for (Group g : groups) {
        for (PartitionNode n : externalGroupInputs(g)) {
            if (isIdentityPartition(n)) {
                Node parent = TridentUtils.getParent(graph, n);
                Group parentGroup = grouper.nodeGroup(parent);
                if (parentGroup != null && !parentGroup.equals(g)) {
                    equivs.addEdge(parentGroup, g);
                }
            }
        }
    }
    Map<Group, Integer> ret = new HashMap<>();
    List<Set<Group>> equivGroups = new ConnectivityInspector<>(equivs).connectedSets();
    for (Set<Group> equivGroup : equivGroups) {
        Integer fixedP = getFixedParallelism(equivGroup);
        Integer maxP = getMaxParallelism(equivGroup);
        if (fixedP != null && maxP != null && maxP < fixedP) {
            throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP);
        }
        Integer p = 1;
        for (Group g : equivGroup) {
            for (Node n : g.nodes) {
                if (n.parallelismHint != null) {
                    p = Math.max(p, n.parallelismHint);
                }
            }
        }
        if (maxP != null)
            p = Math.min(maxP, p);
        if (fixedP != null)
            p = fixedP;
        for (Group g : equivGroup) {
            ret.put(g, p);
        }
    }
    return ret;
}

Example 7 with Group

use of storm.trident.graph.Group in project jstorm by alibaba.

the class TridentTopology method genBoltIds.

private static Map<Group, String> genBoltIds(Collection<Group> groups) {
    Map<Group, String> ret = new HashMap<>();
    int ctr = 0;
    for (Group g : groups) {
        if (!isSpoutGroup(g)) {
            List<String> name = new ArrayList<>();
            name.add("b");
            name.add("" + ctr);
            String groupName = getGroupName(g);
            if (groupName != null && !groupName.isEmpty()) {
                name.add(getGroupName(g));
            }
            ret.put(g, Utils.join(name, "-"));
            ctr++;
        }
    }
    return ret;
}

Example 8 with Group

use of storm.trident.graph.Group in project jstorm by alibaba.

the class TridentTopology method getMaxParallelism.

private static Integer getMaxParallelism(Set<Group> groups) {
    Integer ret = null;
    for (Group g : groups) {
        if (isSpoutGroup(g)) {
            SpoutNode n = (SpoutNode) g.nodes.iterator().next();
            Map conf = getSpoutComponentConfig(n.spout);
            if (conf == null)
                conf = new HashMap();
            Number maxP = (Number) conf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
            if (maxP != null) {
                if (ret == null)
                    ret = maxP.intValue();
                else
                    ret = Math.min(ret, maxP.intValue());
            }
        }
    }
    return ret;
}

Example 9 with Group

use of storm.trident.graph.Group in project storm by nathanmarz.

the class TridentTopology method getMaxParallelism.

private static Integer getMaxParallelism(Set<Group> groups) {
    Integer ret = null;
    for (Group g : groups) {
        if (isSpoutGroup(g)) {
            SpoutNode n = (SpoutNode) g.nodes.iterator().next();
            Map conf = getSpoutComponentConfig(n.spout);
            if (conf == null)
                conf = new HashMap();
            Number maxP = (Number) conf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
            if (maxP != null) {
                if (ret == null)
                    ret = maxP.intValue();
                else
                    ret = Math.min(ret, maxP.intValue());
            }
        }
    }
    return ret;
}

Example 10 with Group

use of storm.trident.graph.Group in project storm by nathanmarz.

the class TridentTopology method getFixedParallelism.

private static Integer getFixedParallelism(Set<Group> groups) {
    Integer ret = null;
    for (Group g : groups) {
        for (Node n : g.nodes) {
            if (n.stateInfo != null && n.stateInfo.spec.requiredNumPartitions != null) {
                int reqPartitions = n.stateInfo.spec.requiredNumPartitions;
                if (ret != null && ret != reqPartitions) {
                    throw new RuntimeException("Cannot have one group have fixed parallelism of two different values");
                }
                ret = reqPartitions;
            }
        }
    }
    return ret;
}