Examples with Node storm.trident.planner.Node used on opensource projects

Example 16 with Node

use of storm.trident.planner.Node in project jstorm by alibaba.

the class TridentTopology method getFixedParallelism.

private static Integer getFixedParallelism(Set<Group> groups) {
    Integer ret = null;
    for (Group g : groups) {
        for (Node n : g.nodes) {
            if (n.stateInfo != null && n.stateInfo.spec.requiredNumPartitions != null) {
                int reqPartitions = n.stateInfo.spec.requiredNumPartitions;
                if (ret != null && ret != reqPartitions) {
                    throw new RuntimeException("Cannot have one group have fixed parallelism of two different values");
                }
                ret = reqPartitions;
            }
        }
    }
    return ret;
}

Example 17 with Node

use of storm.trident.planner.Node in project jstorm by alibaba.

the class TridentTopology method getGroupParallelisms.

private static Map<Group, Integer> getGroupParallelisms(DirectedGraph<Node, IndexedEdge> graph, GraphGrouper grouper, Collection<Group> groups) {
    UndirectedGraph<Group, Object> equivs = new Pseudograph<>(Object.class);
    for (Group g : groups) {
        equivs.addVertex(g);
    }
    for (Group g : groups) {
        for (PartitionNode n : externalGroupInputs(g)) {
            if (isIdentityPartition(n)) {
                Node parent = TridentUtils.getParent(graph, n);
                Group parentGroup = grouper.nodeGroup(parent);
                if (parentGroup != null && !parentGroup.equals(g)) {
                    equivs.addEdge(parentGroup, g);
                }
            }
        }
    }
    Map<Group, Integer> ret = new HashMap<>();
    List<Set<Group>> equivGroups = new ConnectivityInspector<>(equivs).connectedSets();
    for (Set<Group> equivGroup : equivGroups) {
        Integer fixedP = getFixedParallelism(equivGroup);
        Integer maxP = getMaxParallelism(equivGroup);
        if (fixedP != null && maxP != null && maxP < fixedP) {
            throw new RuntimeException("Parallelism is fixed to " + fixedP + " but max parallelism is less than that: " + maxP);
        }
        Integer p = 1;
        for (Group g : equivGroup) {
            for (Node n : g.nodes) {
                if (n.parallelismHint != null) {
                    p = Math.max(p, n.parallelismHint);
                }
            }
        }
        if (maxP != null)
            p = Math.min(maxP, p);
        if (fixedP != null)
            p = fixedP;
        for (Group g : equivGroup) {
            ret.put(g, p);
        }
    }
    return ret;
}

Example 18 with Node

use of storm.trident.planner.Node in project jstorm by alibaba.

the class TridentTopology method completeDRPC.

private static void completeDRPC(DefaultDirectedGraph<Node, IndexedEdge> graph, Map<String, List<Node>> colocate, UniqueIdGen gen) {
    List<Set<Node>> connectedComponents = new ConnectivityInspector<>(graph).connectedSets();
    for (Set<Node> g : connectedComponents) {
        checkValidJoins(g);
    }
    TridentTopology helper = new TridentTopology(graph, colocate, gen);
    for (Set<Node> g : connectedComponents) {
        SpoutNode drpcNode = getDRPCSpoutNode(g);
        if (drpcNode != null) {
            Stream lastStream = new Stream(helper, null, getLastAddedNode(g));
            Stream s = new Stream(helper, null, drpcNode);
            helper.multiReduce(s.project(new Fields("return-info")).batchGlobal(), lastStream.batchGlobal(), new ReturnResultsReducer(), new Fields());
        }
    }
}

Example 19 with Node

use of storm.trident.planner.Node in project jstorm by alibaba.

the class TridentTopology method multiReduce.

public Stream multiReduce(List<Fields> inputFields, List<Stream> streams, MultiReducer function, Fields outputFields) {
    List<String> names = new ArrayList<>();
    for (Stream s : streams) {
        if (s._name != null) {
            names.add(s._name);
        }
    }
    Node n = new ProcessorNode(getUniqueStreamId(), Utils.join(names, "-"), outputFields, outputFields, new MultiReducerProcessor(inputFields, function));
    return addSourcedNode(streams, n);
}

Example 20 with Node

use of storm.trident.planner.Node in project jstorm by alibaba.

the class TridentTopology method checkValidJoins.

private static void checkValidJoins(Collection<Node> g) {
    boolean hasDRPCSpout = false;
    boolean hasBatchSpout = false;
    for (Node n : g) {
        if (n instanceof SpoutNode) {
            SpoutNode.SpoutType type = ((SpoutNode) n).type;
            if (type == SpoutNode.SpoutType.BATCH) {
                hasBatchSpout = true;
            } else if (type == SpoutNode.SpoutType.DRPC) {
                hasDRPCSpout = true;
            }
        }
    }
    if (hasBatchSpout && hasDRPCSpout) {
        throw new RuntimeException("Cannot join DRPC stream with streams originating from other spouts");
    }
}