Examples with ISpout org.apache.storm.spout.ISpout used on opensource projects

Example 1 with ISpout

use of org.apache.storm.spout.ISpout in project storm by apache.

the class SpoutExecutor method ackSpoutMsg.

public void ackSpoutMsg(Executor executor, Task taskData, Long timeDelta, TupleInfo tupleInfo) {
    try {
        ISpout spout = (ISpout) taskData.getTaskObject();
        int taskId = taskData.getTaskId();
        if (executor.getIsDebug()) {
            LOG.info("SPOUT Acking message {} {}", tupleInfo.getId(), tupleInfo.getMessageId());
        }
        spout.ack(tupleInfo.getMessageId());
        new SpoutAckInfo(tupleInfo.getMessageId(), taskId, timeDelta).applyOn(taskData.getUserContext());
        if (timeDelta != null) {
            ((SpoutExecutorStats) executor.getStats()).spoutAckedTuple(tupleInfo.getStream(), timeDelta);
        }
    } catch (Exception e) {
        throw Utils.wrapInRuntime(e);
    }
}

Example 2 with ISpout

use of org.apache.storm.spout.ISpout in project storm by apache.

the class SpoutExecutor method init.

public void init(final ArrayList<Task> idToTask, int idToTaskBase) throws InterruptedException {
    this.threadId = Thread.currentThread().getId();
    executorTransfer.initLocalRecvQueues();
    workerReady.await();
    while (!stormActive.get()) {
        // Topology may be deployed in deactivated mode, wait for activation
        Utils.sleepNoSimulation(100);
    }
    LOG.info("Opening spout {}:{}", componentId, taskIds);
    this.idToTask = idToTask;
    this.maxSpoutPending = ObjectReader.getInt(topoConf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), 0) * idToTask.size();
    this.spouts = new ArrayList<>();
    for (Task task : idToTask) {
        if (task != null) {
            this.spouts.add((ISpout) task.getTaskObject());
        }
    }
    this.pending = new RotatingMap<>(2, new RotatingMap.ExpiredCallback<Long, TupleInfo>() {

        @Override
        public void expire(Long key, TupleInfo tupleInfo) {
            Long timeDelta = null;
            if (tupleInfo.getTimestamp() != 0) {
                timeDelta = Time.deltaMs(tupleInfo.getTimestamp());
            }
            failSpoutMsg(SpoutExecutor.this, idToTask.get(tupleInfo.getTaskId() - idToTaskBase), timeDelta, tupleInfo, "TIMEOUT");
        }
    });
    this.outputCollectors = new ArrayList<>();
    for (int i = 0; i < idToTask.size(); ++i) {
        Task taskData = idToTask.get(i);
        if (taskData == null) {
            continue;
        }
        ISpout spoutObject = (ISpout) taskData.getTaskObject();
        spoutOutputCollector = new SpoutOutputCollectorImpl(spoutObject, this, taskData, emittedCount, hasAckers, rand, hasEventLoggers, isDebug, pending);
        SpoutOutputCollector outputCollector = new SpoutOutputCollector(spoutOutputCollector);
        this.outputCollectors.add(outputCollector);
        if (spoutObject instanceof ICredentialsListener) {
            ((ICredentialsListener) spoutObject).setCredentials(credentials);
        }
        spoutObject.open(topoConf, taskData.getUserContext(), outputCollector);
    }
    openOrPrepareWasCalled.set(true);
    LOG.info("Opened spout {}:{}", componentId, taskIds);
    setupTicks(true);
    setupMetrics();
}

Example 3 with ISpout

use of org.apache.storm.spout.ISpout in project storm by apache.

the class SpoutExecutor method failSpoutMsg.

public void failSpoutMsg(SpoutExecutor executor, Task taskData, Long timeDelta, TupleInfo tupleInfo, String reason) {
    try {
        ISpout spout = (ISpout) taskData.getTaskObject();
        int taskId = taskData.getTaskId();
        if (executor.getIsDebug()) {
            LOG.info("SPOUT Failing {} : {} REASON: {}", tupleInfo.getRootId(), tupleInfo, reason);
        }
        spout.fail(tupleInfo.getMessageId());
        new SpoutFailInfo(tupleInfo.getMessageId(), taskId, timeDelta).applyOn(taskData.getUserContext());
        if (timeDelta != null) {
            executor.getStats().spoutFailedTuple(tupleInfo.getStream());
            taskData.getTaskMetrics().spoutFailedTuple(tupleInfo.getStream());
        }
    } catch (Exception e) {
        throw Utils.wrapInRuntime(e);
    }
}

Example 4 with ISpout

use of org.apache.storm.spout.ISpout in project storm by apache.

the class SpoutExecutor method init.

public void init(final Map<Integer, Task> idToTask) {
    while (!stormActive.get()) {
        Utils.sleep(100);
    }
    LOG.info("Opening spout {}:{}", componentId, idToTask.keySet());
    this.idToTask = idToTask;
    this.maxSpoutPending = Utils.getInt(stormConf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), 0) * idToTask.size();
    this.spouts = new ArrayList<>();
    for (Task task : idToTask.values()) {
        this.spouts.add((ISpout) task.getTaskObject());
    }
    this.pending = new RotatingMap<>(2, new RotatingMap.ExpiredCallback<Long, TupleInfo>() {

        @Override
        public void expire(Long key, TupleInfo tupleInfo) {
            Long timeDelta = null;
            if (tupleInfo.getTimestamp() != 0) {
                timeDelta = Time.deltaMs(tupleInfo.getTimestamp());
            }
            failSpoutMsg(SpoutExecutor.this, idToTask.get(tupleInfo.getTaskId()), timeDelta, tupleInfo, "TIMEOUT");
        }
    });
    this.spoutThrottlingMetrics.registerAll(stormConf, idToTask.values().iterator().next().getUserContext());
    this.outputCollectors = new ArrayList<>();
    for (Map.Entry<Integer, Task> entry : idToTask.entrySet()) {
        Task taskData = entry.getValue();
        ISpout spoutObject = (ISpout) taskData.getTaskObject();
        SpoutOutputCollectorImpl spoutOutputCollector = new SpoutOutputCollectorImpl(spoutObject, this, taskData, entry.getKey(), emittedCount, hasAckers, rand, hasEventLoggers, isDebug, pending);
        SpoutOutputCollector outputCollector = new SpoutOutputCollector(spoutOutputCollector);
        this.outputCollectors.add(outputCollector);
        taskData.getBuiltInMetrics().registerAll(stormConf, taskData.getUserContext());
        Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue);
        BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, taskData.getUserContext());
        if (spoutObject instanceof ICredentialsListener) {
            ((ICredentialsListener) spoutObject).setCredentials(credentials);
        }
        spoutObject.open(stormConf, taskData.getUserContext(), outputCollector);
    }
    openOrPrepareWasCalled.set(true);
    LOG.info("Opened spout {}:{}", componentId, idToTask.keySet());
    setupMetrics();
}

Example 5 with ISpout

use of org.apache.storm.spout.ISpout in project storm by apache.

the class SpoutExecutor method failSpoutMsg.

public void failSpoutMsg(Executor executor, Task taskData, Long timeDelta, TupleInfo tupleInfo, String reason) {
    try {
        ISpout spout = (ISpout) taskData.getTaskObject();
        int taskId = taskData.getTaskId();
        if (executor.getIsDebug()) {
            LOG.info("SPOUT Failing {} : {} REASON: {}", tupleInfo.getId(), tupleInfo, reason);
        }
        spout.fail(tupleInfo.getMessageId());
        new SpoutFailInfo(tupleInfo.getMessageId(), taskId, timeDelta).applyOn(taskData.getUserContext());
        if (timeDelta != null) {
            ((SpoutExecutorStats) executor.getStats()).spoutFailedTuple(tupleInfo.getStream(), timeDelta);
        }
    } catch (Exception e) {
        throw Utils.wrapInRuntime(e);
    }
}