Examples with Node org.apache.storm.trident.planner.Node used on opensource projects

Example 11 with Node

use of org.apache.storm.trident.planner.Node in project storm by apache.

the class TridentTopology method getGroupParallelisms.

private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) {
    UndirectedGraph<Group, Object> equivs = new Pseudograph<>(Object.class);
    for (Group g : groups) {
        equivs.addVertex(g);
    }
    for (Group g : groups) {
        for (PartitionNode n : externalGroupInputs(g)) {
            if (isIdentityPartition(n)) {
                Node parent = TridentUtils.getParent(graph, n);
                Group parentGroup = grouper.nodeGroup(parent);
                if (parentGroup != null && !parentGroup.equals(g)) {
                    equivs.addEdge(parentGroup, g);
                }
            }
        }
    }
    Map<Group, Integer> ret = new HashMap<>();
    List<Set<Group>> equivGroups = new ConnectivityInspector<>(equivs).connectedSets();
    for (Set<Group> equivGroup : equivGroups) {
        Integer fixedP = getFixedParallelism(equivGroup);
        Integer maxP = getMaxParallelism(equivGroup);
        if (fixedP != null && maxP != null && maxP < fixedP) {
            throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP);
        }
        Integer p = 1;
        for (Group g : equivGroup) {
            for (Node n : g.nodes) {
                if (n.parallelismHint != null) {
                    p = Math.max(p, n.parallelismHint);
                }
            }
        }
        if (maxP != null)
            p = Math.min(maxP, p);
        if (fixedP != null)
            p = fixedP;
        for (Group g : equivGroup) {
            ret.put(g, p);
        }
    }
    return ret;
}

Example 12 with Node

use of org.apache.storm.trident.planner.Node in project storm by apache.

the class TridentTopology method newDRPCStream.

private Stream newDRPCStream(DRPCSpout spout) {
    // TODO: consider adding a shuffle grouping after the spout to avoid so much routing of the args/return-info all over the place
    // (at least until its possible to just pack bolt logic into the spout itself)
    Node n = new SpoutNode(getUniqueStreamId(), TridentUtils.getSingleOutputStreamFields(spout), null, spout, SpoutNode.SpoutType.DRPC);
    Stream nextStream = addNode(n);
    // later on, this will be joined back with return-info and all the results
    return nextStream.project(new Fields("args"));
}

Example 13 with Node

use of org.apache.storm.trident.planner.Node in project storm by apache.

the class TridentTopology method getFixedParallelism.

private static Integer getFixedParallelism(Set<Group> groups) {
    Integer ret = null;
    for (Group g : groups) {
        for (Node n : g.nodes) {
            if (n.stateInfo != null && n.stateInfo.spec.requiredNumPartitions != null) {
                int reqPartitions = n.stateInfo.spec.requiredNumPartitions;
                if (ret != null && ret != reqPartitions) {
                    throw new RuntimeException("Cannot have one group have fixed parallelism of two different values");
                }
                ret = reqPartitions;
            }
        }
    }
    return ret;
}