Examples with OutputCollector backtype.storm.task.OutputCollector used on opensource projects

Example 1 with OutputCollector

use of backtype.storm.task.OutputCollector in project storm by nathanmarz.

the class TridentBoltExecutor method prepare.

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    _messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L;
    _lastRotate = System.currentTimeMillis();
    _batches = new RotatingMap(2);
    _context = context;
    _collector = collector;
    _coordCollector = new CoordinatedOutputCollector(collector);
    _coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(_coordCollector));
    _coordConditions = (Map) context.getExecutorData("__coordConditions");
    if (_coordConditions == null) {
        _coordConditions = new HashMap();
        for (String batchGroup : _coordSpecs.keySet()) {
            CoordSpec spec = _coordSpecs.get(batchGroup);
            CoordCondition cond = new CoordCondition();
            cond.commitStream = spec.commitStream;
            cond.expectedTaskReports = 0;
            for (String comp : spec.coords.keySet()) {
                CoordType ct = spec.coords.get(comp);
                if (ct.equals(CoordType.single())) {
                    cond.expectedTaskReports += 1;
                } else {
                    cond.expectedTaskReports += context.getComponentTasks(comp).size();
                }
            }
            cond.targetTasks = new HashSet<Integer>();
            for (String component : Utils.get(context.getThisTargets(), COORD_STREAM(batchGroup), new HashMap<String, Grouping>()).keySet()) {
                cond.targetTasks.addAll(context.getComponentTasks(component));
            }
            _coordConditions.put(batchGroup, cond);
        }
        context.setExecutorData("_coordConditions", _coordConditions);
    }
    _bolt.prepare(conf, context, _coordOutputCollector);
}

Example 2 with OutputCollector

use of backtype.storm.task.OutputCollector in project jstorm by alibaba.

the class CoordinatedBolt method prepare.

public void prepare(Map config, TopologyContext context, OutputCollector collector) {
    TimeCacheMap.ExpiredCallback<Object, TrackingInfo> callback = null;
    if (_delegate instanceof TimeoutCallback) {
        callback = new TimeoutItems();
    }
    _tracked = new TimeCacheMap<Object, TrackingInfo>(context.maxTopologyMessageTimeout(), callback);
    _collector = collector;
    _delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector)));
    for (String component : Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap<String, Grouping>()).keySet()) {
        for (Integer task : context.getComponentTasks(component)) {
            _countOutTasks.add(task);
        }
    }
    if (!_sourceArgs.isEmpty()) {
        _numSourceReports = 0;
        for (Entry<String, SourceArgs> entry : _sourceArgs.entrySet()) {
            if (entry.getValue().singleCount) {
                _numSourceReports += 1;
            } else {
                _numSourceReports += context.getComponentTasks(entry.getKey()).size();
            }
        }
    }
}

Example 3 with OutputCollector

use of backtype.storm.task.OutputCollector in project jstorm by alibaba.

the class TransactionBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.conf = stormConf;
    this.topologyContext = context;
    this.topologyId = topologyContext.getTopologyId();
    this.taskId = topologyContext.getThisTaskId();
    this.componentId = topologyContext.getThisComponentId();
    this.upstreamTasks = TransactionCommon.getUpstreamTasks(componentId, topologyContext);
    this.downstreamTasks = TransactionCommon.getDownstreamTasks(componentId, topologyContext);
    this.topologyMasterId = context.getTopologyMasterId();
    LOG.info("TransactionBolt: upstreamTasks=" + upstreamTasks + ", downstreamTasks=" + downstreamTasks);
    this.outputCollector = new TransactionOutputCollector(this, collector);
    this.boltExecutor.prepare(conf, context, new OutputCollector(outputCollector));
    this.boltStatus = State.INIT;
    if (sysTopology == null) {
        try {
            sysTopology = Common.system_topology(stormConf, context.getRawTopology());
        } catch (InvalidTopologyException e) {
            LOG.error("Failed to build system topology", e);
            throw new RuntimeException(e);
        }
    }
    this.lastSuccessfulBatch = new ConcurrentHashMap<Integer, Long>();
    this.processingBatches = new HashMap<Integer, Map<Long, BatchTracker>>();
    Set<String> upstreamSpoutNames = TransactionCommon.getUpstreamSpouts(componentId, topologyContext);
    for (String spoutName : upstreamSpoutNames) {
        int groupId = TransactionCommon.groupIndex(topologyContext.getRawTopology(), spoutName);
        lastSuccessfulBatch.put(groupId, TransactionCommon.INIT_BATCH_ID);
        processingBatches.put(groupId, new HashMap<Long, BatchTracker>());
    }
    this.batchCache = new BatchCache(context, upstreamSpoutNames, sysTopology);
    this.kryoInput = new Input(1);
    this.streamIds = new SerializationFactory.IdDictionary(sysTopology);
    this.inputStreamIds = new HashSet<Integer>();
    Set<GlobalStreamId> inputs = topologyContext.getThisSources().keySet();
    for (GlobalStreamId stream : inputs) {
        inputStreamIds.add(streamIds.getStreamId(stream.get_componentId(), stream.get_streamId()));
    }
    for (String upstreamComponentId : TransactionCommon.getUpstreamComponents(componentId, topologyContext)) {
        inputStreamIds.add(streamIds.getStreamId(upstreamComponentId, TransactionCommon.BARRIER_STREAM_ID));
    }
    //LOG.info("Stream info prepare: streamIds={}, inputStreams={}, inputStreamIds={}", streamIds, inputs, inputStreamIds);
    startInitState();
}

Example 4 with OutputCollector

use of backtype.storm.task.OutputCollector in project pulsar by yahoo.

the class PulsarBoltTest method testSharedProducer.

@Test
public void testSharedProducer() throws Exception {
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.publishers.size(), 1);
    PulsarBolt otherBolt = new PulsarBolt(pulsarBoltConf, new ClientConfiguration());
    MockOutputCollector otherMockCollector = new MockOutputCollector();
    OutputCollector collector = new OutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-bolt-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherBolt.prepare(Maps.newHashMap(), context, collector);
    topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.publishers.size(), 1);
    otherBolt.close();
    topicStats = admin.persistentTopics().getStats(topic);
    Assert.assertEquals(topicStats.publishers.size(), 1);
}

Example 5 with OutputCollector

use of backtype.storm.task.OutputCollector in project jstorm by alibaba.

the class TridentBoltExecutor method prepare.

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    _messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L;
    _lastRotate = System.currentTimeMillis();
    _batches = new RotatingMap<>(2);
    _context = context;
    _collector = collector;
    _coordCollector = new CoordinatedOutputCollector(new OutputCollector(collector));
    _coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(_coordCollector));
    _coordConditions = (Map) context.getExecutorData("__coordConditions");
    if (_coordConditions == null) {
        _coordConditions = new HashMap<>();
        for (String batchGroup : _coordSpecs.keySet()) {
            CoordSpec spec = _coordSpecs.get(batchGroup);
            CoordCondition cond = new CoordCondition();
            cond.commitStream = spec.commitStream;
            cond.expectedTaskReports = 0;
            for (String comp : spec.coords.keySet()) {
                CoordType ct = spec.coords.get(comp);
                if (ct.equals(CoordType.single())) {
                    cond.expectedTaskReports += 1;
                } else {
                    cond.expectedTaskReports += context.getComponentTasks(comp).size();
                }
            }
            cond.targetTasks = new HashSet<>();
            for (String component : Utils.get(context.getThisTargets(), COORD_STREAM(batchGroup), new HashMap<String, Grouping>()).keySet()) {
                cond.targetTasks.addAll(context.getComponentTasks(component));
            }
            _coordConditions.put(batchGroup, cond);
        }
        context.setExecutorData("_coordConditions", _coordConditions);
    }
    _bolt.prepare(conf, context, _coordOutputCollector);
}