Examples with SpoutOutputCollector org.apache.storm.spout.SpoutOutputCollector used on opensource projects

Example 1 with SpoutOutputCollector

use of org.apache.storm.spout.SpoutOutputCollector in project flink by apache.

the class SpoutWrapper method run.

@Override
public final void run(final SourceContext<OUT> ctx) throws Exception {
    final GlobalJobParameters config = super.getRuntimeContext().getExecutionConfig().getGlobalJobParameters();
    StormConfig stormConfig = new StormConfig();
    if (config != null) {
        if (config instanceof StormConfig) {
            stormConfig = (StormConfig) config;
        } else {
            stormConfig.putAll(config.toMap());
        }
    }
    final TopologyContext stormTopologyContext = WrapperSetupHelper.createTopologyContext((StreamingRuntimeContext) super.getRuntimeContext(), this.spout, this.name, this.stormTopology, stormConfig);
    SpoutCollector<OUT> collector = new SpoutCollector<OUT>(this.numberOfAttributes, stormTopologyContext.getThisTaskId(), ctx);
    this.spout.open(stormConfig, stormTopologyContext, new SpoutOutputCollector(collector));
    this.spout.activate();
    if (numberOfInvocations == null) {
        if (this.spout instanceof FiniteSpout) {
            final FiniteSpout finiteSpout = (FiniteSpout) this.spout;
            while (this.isRunning && !finiteSpout.reachedEnd()) {
                finiteSpout.nextTuple();
            }
        } else {
            while (this.isRunning) {
                this.spout.nextTuple();
            }
        }
    } else {
        int counter = this.numberOfInvocations;
        if (counter >= 0) {
            while ((--counter >= 0) && this.isRunning) {
                this.spout.nextTuple();
            }
        } else {
            do {
                collector.tupleEmitted = false;
                this.spout.nextTuple();
            } while (collector.tupleEmitted && this.isRunning);
        }
    }
}

Example 2 with SpoutOutputCollector

use of org.apache.storm.spout.SpoutOutputCollector in project storm by apache.

the class SpoutTracker method open.

public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _tracker = new SpoutTrackOutputCollector(collector);
    _delegate.open(conf, context, new SpoutOutputCollector(_tracker));
}

Example 3 with SpoutOutputCollector

use of org.apache.storm.spout.SpoutOutputCollector in project storm by apache.

the class JmsSpoutTest method testFailure.

@Test
public void testFailure() throws JMSException, Exception {
    JmsSpout spout = new JmsSpout();
    JmsProvider mockProvider = new MockJmsProvider();
    MockSpoutOutputCollector mockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(mockCollector);
    spout.setJmsProvider(new MockJmsProvider());
    spout.setJmsTupleProducer(new MockTupleProducer());
    spout.setJmsAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
    // Rapid recovery for testing.
    spout.setRecoveryPeriod(10);
    spout.open(new HashMap<String, String>(), null, collector);
    Message msg = this.sendMessage(mockProvider.connectionFactory(), mockProvider.destination());
    Thread.sleep(100);
    // Pretend to be storm.
    spout.nextTuple();
    Assert.assertTrue(mockCollector.emitted);
    mockCollector.reset();
    // Mock failure
    spout.fail(msg.getJMSMessageID());
    Thread.sleep(5000);
    // Pretend to be storm.
    spout.nextTuple();
    Thread.sleep(5000);
    // Should have been re-emitted
    Assert.assertTrue(mockCollector.emitted);
}

Example 4 with SpoutOutputCollector

use of org.apache.storm.spout.SpoutOutputCollector in project storm by apache.

the class SpoutExecutor method init.

public void init(final Map<Integer, Task> idToTask) {
    while (!stormActive.get()) {
        Utils.sleep(100);
    }
    LOG.info("Opening spout {}:{}", componentId, idToTask.keySet());
    this.idToTask = idToTask;
    this.maxSpoutPending = Utils.getInt(stormConf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), 0) * idToTask.size();
    this.spouts = new ArrayList<>();
    for (Task task : idToTask.values()) {
        this.spouts.add((ISpout) task.getTaskObject());
    }
    this.pending = new RotatingMap<>(2, new RotatingMap.ExpiredCallback<Long, TupleInfo>() {

        @Override
        public void expire(Long key, TupleInfo tupleInfo) {
            Long timeDelta = null;
            if (tupleInfo.getTimestamp() != 0) {
                timeDelta = Time.deltaMs(tupleInfo.getTimestamp());
            }
            failSpoutMsg(SpoutExecutor.this, idToTask.get(tupleInfo.getTaskId()), timeDelta, tupleInfo, "TIMEOUT");
        }
    });
    this.spoutThrottlingMetrics.registerAll(stormConf, idToTask.values().iterator().next().getUserContext());
    this.outputCollectors = new ArrayList<>();
    for (Map.Entry<Integer, Task> entry : idToTask.entrySet()) {
        Task taskData = entry.getValue();
        ISpout spoutObject = (ISpout) taskData.getTaskObject();
        SpoutOutputCollectorImpl spoutOutputCollector = new SpoutOutputCollectorImpl(spoutObject, this, taskData, entry.getKey(), emittedCount, hasAckers, rand, hasEventLoggers, isDebug, pending);
        SpoutOutputCollector outputCollector = new SpoutOutputCollector(spoutOutputCollector);
        this.outputCollectors.add(outputCollector);
        taskData.getBuiltInMetrics().registerAll(stormConf, taskData.getUserContext());
        Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue);
        BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, taskData.getUserContext());
        if (spoutObject instanceof ICredentialsListener) {
            ((ICredentialsListener) spoutObject).setCredentials(credentials);
        }
        spoutObject.open(stormConf, taskData.getUserContext(), outputCollector);
    }
    openOrPrepareWasCalled.set(true);
    LOG.info("Opened spout {}:{}", componentId, idToTask.keySet());
    setupMetrics();
}

Example 5 with SpoutOutputCollector

use of org.apache.storm.spout.SpoutOutputCollector in project storm by apache.

the class RichSpoutBatchTriggerer method open.

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _delegate.open(conf, context, new SpoutOutputCollector(new StreamOverrideCollector(collector)));
    _outputTasks = new ArrayList<>();
    for (String component : Utils.get(context.getThisTargets(), _coordStream, new HashMap<String, Grouping>()).keySet()) {
        _outputTasks.addAll(context.getComponentTasks(component));
    }
    _rand = new Random(Utils.secureRandomLong());
}