Examples with DirectedSubgraph org.jgrapht.graph.DirectedSubgraph used on opensource projects

Example 1 with DirectedSubgraph

use of org.jgrapht.graph.DirectedSubgraph in project storm by nathanmarz.

the class SubtopologyBolt method prepare.

@Override
public void prepare(Map conf, TopologyContext context, BatchOutputCollector batchCollector) {
    int thisComponentNumTasks = context.getComponentTasks(context.getThisComponentId()).size();
    for (Node n : _nodes) {
        if (n.stateInfo != null) {
            State s = n.stateInfo.spec.stateFactory.makeState(conf, context, context.getThisTaskIndex(), thisComponentNumTasks);
            context.setTaskData(n.stateInfo.id, s);
        }
    }
    DirectedSubgraph<Node, Object> subgraph = new DirectedSubgraph(_graph, _nodes, null);
    TopologicalOrderIterator it = new TopologicalOrderIterator<Node, Object>(subgraph);
    int stateIndex = 0;
    while (it.hasNext()) {
        Node n = (Node) it.next();
        if (n instanceof ProcessorNode) {
            ProcessorNode pn = (ProcessorNode) n;
            String batchGroup = _batchGroups.get(n);
            if (!_myTopologicallyOrdered.containsKey(batchGroup)) {
                _myTopologicallyOrdered.put(batchGroup, new ArrayList());
            }
            _myTopologicallyOrdered.get(batchGroup).add(pn.processor);
            List<String> parentStreams = new ArrayList();
            List<Factory> parentFactories = new ArrayList();
            for (Node p : TridentUtils.getParents(_graph, n)) {
                parentStreams.add(p.streamId);
                if (_nodes.contains(p)) {
                    parentFactories.add(_outputFactories.get(p));
                } else {
                    if (!_roots.containsKey(p.streamId)) {
                        _roots.put(p.streamId, new InitialReceiver(p.streamId, getSourceOutputFields(context, p.streamId)));
                    }
                    _roots.get(p.streamId).addReceiver(pn.processor);
                    parentFactories.add(_roots.get(p.streamId).getOutputFactory());
                }
            }
            List<TupleReceiver> targets = new ArrayList();
            boolean outgoingNode = false;
            for (Node cn : TridentUtils.getChildren(_graph, n)) {
                if (_nodes.contains(cn)) {
                    targets.add(((ProcessorNode) cn).processor);
                } else {
                    outgoingNode = true;
                }
            }
            if (outgoingNode) {
                targets.add(new BridgeReceiver(batchCollector));
            }
            TridentContext triContext = new TridentContext(pn.selfOutFields, parentFactories, parentStreams, targets, pn.streamId, stateIndex, batchCollector);
            pn.processor.prepare(conf, context, triContext);
            _outputFactories.put(n, pn.processor.getOutputFactory());
        }
        stateIndex++;
    }
// TODO: get prepared one time into executor data... need to avoid the ser/deser
// for each task (probably need storm to support boltfactory)
}

Example 2 with DirectedSubgraph

use of org.jgrapht.graph.DirectedSubgraph in project jstorm by alibaba.

the class SubtopologyBolt method prepare.

@Override
public void prepare(Map conf, TopologyContext context, BatchOutputCollector batchCollector) {
    int thisComponentNumTasks = context.getComponentTasks(context.getThisComponentId()).size();
    for (Node n : _nodes) {
        if (n.stateInfo != null) {
            State s = n.stateInfo.spec.stateFactory.makeState(conf, context, context.getThisTaskIndex(), thisComponentNumTasks);
            context.setTaskData(n.stateInfo.id, s);
        }
    }
    DirectedSubgraph<Node, Object> subgraph = new DirectedSubgraph(_graph, _nodes, null);
    TopologicalOrderIterator it = new TopologicalOrderIterator<Node, Object>(subgraph);
    int stateIndex = 0;
    while (it.hasNext()) {
        Node n = (Node) it.next();
        if (n instanceof ProcessorNode) {
            ProcessorNode pn = (ProcessorNode) n;
            String batchGroup = _batchGroups.get(n);
            if (!_myTopologicallyOrdered.containsKey(batchGroup)) {
                _myTopologicallyOrdered.put(batchGroup, new ArrayList());
            }
            _myTopologicallyOrdered.get(batchGroup).add(pn.processor);
            List<String> parentStreams = new ArrayList();
            List<Factory> parentFactories = new ArrayList();
            for (Node p : TridentUtils.getParents(_graph, n)) {
                parentStreams.add(p.streamId);
                if (_nodes.contains(p)) {
                    parentFactories.add(_outputFactories.get(p));
                } else {
                    if (!_roots.containsKey(p.streamId)) {
                        _roots.put(p.streamId, new InitialReceiver(p.streamId, getSourceOutputFields(context, p.streamId)));
                    }
                    _roots.get(p.streamId).addReceiver(pn.processor);
                    parentFactories.add(_roots.get(p.streamId).getOutputFactory());
                }
            }
            List<TupleReceiver> targets = new ArrayList();
            boolean outgoingNode = false;
            for (Node cn : TridentUtils.getChildren(_graph, n)) {
                if (_nodes.contains(cn)) {
                    targets.add(((ProcessorNode) cn).processor);
                } else {
                    outgoingNode = true;
                }
            }
            if (outgoingNode) {
                targets.add(new BridgeReceiver(batchCollector));
            }
            TridentContext triContext = new TridentContext(pn.selfOutFields, parentFactories, parentStreams, targets, pn.streamId, stateIndex, batchCollector);
            pn.processor.prepare(conf, context, triContext);
            _outputFactories.put(n, pn.processor.getOutputFactory());
        }
        stateIndex++;
    }
// TODO: get prepared one time into executor data... need to avoid the ser/deser
// for each task (probably need storm to support boltfactory)
}

Example 3 with DirectedSubgraph

use of org.jgrapht.graph.DirectedSubgraph in project storm by apache.

the class SubtopologyBolt method prepare.

@Override
public void prepare(Map conf, TopologyContext context, BatchOutputCollector batchCollector) {
    int thisComponentNumTasks = context.getComponentTasks(context.getThisComponentId()).size();
    for (Node n : _nodes) {
        if (n.stateInfo != null) {
            State s = n.stateInfo.spec.stateFactory.makeState(conf, context, context.getThisTaskIndex(), thisComponentNumTasks);
            context.setTaskData(n.stateInfo.id, s);
        }
    }
    DirectedSubgraph<Node, Object> subgraph = new DirectedSubgraph(_graph, _nodes, null);
    TopologicalOrderIterator it = new TopologicalOrderIterator<>(subgraph);
    int stateIndex = 0;
    while (it.hasNext()) {
        Node n = (Node) it.next();
        if (n instanceof ProcessorNode) {
            ProcessorNode pn = (ProcessorNode) n;
            String batchGroup = _batchGroups.get(n);
            if (!_myTopologicallyOrdered.containsKey(batchGroup)) {
                _myTopologicallyOrdered.put(batchGroup, new ArrayList());
            }
            _myTopologicallyOrdered.get(batchGroup).add(pn.processor);
            List<String> parentStreams = new ArrayList<>();
            List<Factory> parentFactories = new ArrayList<>();
            for (Node p : TridentUtils.getParents(_graph, n)) {
                parentStreams.add(p.streamId);
                if (_nodes.contains(p)) {
                    parentFactories.add(_outputFactories.get(p));
                } else {
                    if (!_roots.containsKey(p.streamId)) {
                        _roots.put(p.streamId, new InitialReceiver(p.streamId, getSourceOutputFields(context, p.streamId)));
                    }
                    _roots.get(p.streamId).addReceiver(pn.processor);
                    parentFactories.add(_roots.get(p.streamId).getOutputFactory());
                }
            }
            List<TupleReceiver> targets = new ArrayList<>();
            boolean outgoingNode = false;
            for (Node cn : TridentUtils.getChildren(_graph, n)) {
                if (_nodes.contains(cn)) {
                    targets.add(((ProcessorNode) cn).processor);
                } else {
                    outgoingNode = true;
                }
            }
            if (outgoingNode) {
                targets.add(new BridgeReceiver(batchCollector));
            }
            TridentContext triContext = new TridentContext(pn.selfOutFields, parentFactories, parentStreams, targets, pn.streamId, stateIndex, batchCollector);
            pn.processor.prepare(conf, context, triContext);
            _outputFactories.put(n, pn.processor.getOutputFactory());
        }
        stateIndex++;
    }
// TODO: get prepared one time into executor data... need to avoid the ser/deser
// for each task (probably need storm to support boltfactory)
}

Example 4 with DirectedSubgraph

use of org.jgrapht.graph.DirectedSubgraph in project storm by apache.

the class ProcessorBoltDelegate method prepare.

void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.stormConf = stormConf;
    topologyContext = context;
    outputCollector = collector;
    DirectedSubgraph<Node, Edge> subgraph = new DirectedSubgraph<>(graph, new HashSet<>(nodes), null);
    TopologicalOrderIterator<Node, Edge> it = new TopologicalOrderIterator<>(subgraph);
    while (it.hasNext()) {
        Node node = it.next();
        if (!(node instanceof ProcessorNode)) {
            throw new IllegalStateException("Not a processor node " + node);
        }
        ProcessorNode processorNode = (ProcessorNode) node;
        List<ProcessorNode> children = StreamUtil.getChildren(subgraph, processorNode);
        ProcessorContext processorContext;
        if (children.isEmpty()) {
            processorContext = createEmittingContext(processorNode);
        } else {
            Multimap<String, ProcessorNode> streamToChildren = ArrayListMultimap.create();
            for (ProcessorNode child : children) {
                for (String stream : child.getParentStreams(processorNode)) {
                    streamToChildren.put(stream, child);
                }
            }
            ForwardingProcessorContext forwardingContext = new ForwardingProcessorContext(processorNode, streamToChildren);
            if (hasOutgoingChild(processorNode, new HashSet<>(children))) {
                processorContext = new ChainedProcessorContext(processorNode, forwardingContext, createEmittingContext(processorNode));
            } else {
                processorContext = forwardingContext;
            }
        }
        processorNode.initProcessorContext(processorContext);
    }
    if (timestampField != null) {
        for (EmittingProcessorContext ctx : emittingProcessorContexts) {
            ctx.setTimestampField(timestampField);
        }
    }
    for (String stream : streamToInitialProcessors.keySet()) {
        streamToInputTaskCount.put(stream, getStreamInputTaskCount(context, stream));
    }
}