Examples with Task org.apache.storm.daemon.Task used on opensource projects

Example 1 with Task

use of org.apache.storm.daemon.Task in project storm by apache.

the class Executor method mkExecutor.

public static Executor mkExecutor(WorkerState workerState, List<Long> executorId, Map<String, String> credentials) {
    Executor executor;
    WorkerTopologyContext workerTopologyContext = workerState.getWorkerTopologyContext();
    List<Integer> taskIds = StormCommon.executorIdToTasks(executorId);
    String componentId = workerTopologyContext.getComponentId(taskIds.get(0));
    String type = getExecutorType(workerTopologyContext, componentId);
    if (StatsUtil.SPOUT.equals(type)) {
        executor = new SpoutExecutor(workerState, executorId, credentials);
        executor.stats = new SpoutExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()));
    } else {
        executor = new BoltExecutor(workerState, executorId, credentials);
        executor.stats = new BoltExecutorStats(ConfigUtils.samplingRate(executor.getStormConf()));
    }
    Map<Integer, Task> idToTask = new HashMap<>();
    for (Integer taskId : taskIds) {
        try {
            Task task = new Task(executor, taskId);
            executor.sendUnanchored(task, StormCommon.SYSTEM_STREAM_ID, new Values("startup"), executor.getExecutorTransfer());
            idToTask.put(taskId, task);
        } catch (IOException ex) {
            throw Utils.wrapInRuntime(ex);
        }
    }
    executor.idToTask = idToTask;
    return executor;
}

Example 2 with Task

use of org.apache.storm.daemon.Task in project storm by apache.

the class ExecutorShutdown method shutdown.

@Override
public void shutdown() {
    try {
        LOG.info("Shutting down executor " + executor.getComponentId() + ":" + executor.getExecutorId());
        executor.getReceiveQueue().haltWithInterrupt();
        executor.getTransferWorkerQueue().haltWithInterrupt();
        for (Utils.SmartThread t : threads) {
            t.interrupt();
        }
        for (Utils.SmartThread t : threads) {
            LOG.debug("Executor " + executor.getComponentId() + ":" + executor.getExecutorId() + " joining thread " + t.getName());
            t.join();
        }
        executor.getStats().cleanupStats();
        for (Task task : taskDatas.values()) {
            TopologyContext userContext = task.getUserContext();
            for (ITaskHook hook : userContext.getHooks()) {
                hook.cleanup();
            }
        }
        executor.getStormClusterState().disconnect();
        if (executor.getOpenOrPrepareWasCalled().get()) {
            for (Task task : taskDatas.values()) {
                Object object = task.getTaskObject();
                if (object instanceof ISpout) {
                    ((ISpout) object).close();
                } else if (object instanceof IBolt) {
                    ((IBolt) object).cleanup();
                } else {
                    LOG.error("unknown component object");
                }
            }
        }
        LOG.info("Shut down executor " + executor.getComponentId() + ":" + executor.getExecutorId());
    } catch (Exception e) {
        throw Utils.wrapInRuntime(e);
    }
}

Example 3 with Task

use of org.apache.storm.daemon.Task in project storm by apache.

the class BoltExecutor method init.

public void init(Map<Integer, Task> idToTask) {
    while (!stormActive.get()) {
        Utils.sleep(100);
    }
    LOG.info("Preparing bolt {}:{}", componentId, idToTask.keySet());
    for (Map.Entry<Integer, Task> entry : idToTask.entrySet()) {
        Task taskData = entry.getValue();
        IBolt boltObject = (IBolt) taskData.getTaskObject();
        TopologyContext userContext = taskData.getUserContext();
        taskData.getBuiltInMetrics().registerAll(stormConf, userContext);
        if (boltObject instanceof ICredentialsListener) {
            ((ICredentialsListener) boltObject).setCredentials(credentials);
        }
        if (Constants.SYSTEM_COMPONENT_ID.equals(componentId)) {
            Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue, "transfer", workerData.getTransferQueue());
            BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, userContext);
            Map cachedNodePortToSocket = (Map) workerData.getCachedNodeToPortSocket().get();
            BuiltinMetricsUtil.registerIconnectionClientMetrics(cachedNodePortToSocket, stormConf, userContext);
            BuiltinMetricsUtil.registerIconnectionServerMetric(workerData.getReceiver(), stormConf, userContext);
        } else {
            Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue);
            BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, userContext);
        }
        IOutputCollector outputCollector = new BoltOutputCollectorImpl(this, taskData, entry.getKey(), rand, hasEventLoggers, isDebug);
        boltObject.prepare(stormConf, userContext, new OutputCollector(outputCollector));
    }
    openOrPrepareWasCalled.set(true);
    LOG.info("Prepared bolt {}:{}", componentId, idToTask.keySet());
    setupMetrics();
}

Example 4 with Task

use of org.apache.storm.daemon.Task in project storm by apache.

the class SpoutExecutor method init.

public void init(final Map<Integer, Task> idToTask) {
    while (!stormActive.get()) {
        Utils.sleep(100);
    }
    LOG.info("Opening spout {}:{}", componentId, idToTask.keySet());
    this.idToTask = idToTask;
    this.maxSpoutPending = Utils.getInt(stormConf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), 0) * idToTask.size();
    this.spouts = new ArrayList<>();
    for (Task task : idToTask.values()) {
        this.spouts.add((ISpout) task.getTaskObject());
    }
    this.pending = new RotatingMap<>(2, new RotatingMap.ExpiredCallback<Long, TupleInfo>() {

        @Override
        public void expire(Long key, TupleInfo tupleInfo) {
            Long timeDelta = null;
            if (tupleInfo.getTimestamp() != 0) {
                timeDelta = Time.deltaMs(tupleInfo.getTimestamp());
            }
            failSpoutMsg(SpoutExecutor.this, idToTask.get(tupleInfo.getTaskId()), timeDelta, tupleInfo, "TIMEOUT");
        }
    });
    this.spoutThrottlingMetrics.registerAll(stormConf, idToTask.values().iterator().next().getUserContext());
    this.outputCollectors = new ArrayList<>();
    for (Map.Entry<Integer, Task> entry : idToTask.entrySet()) {
        Task taskData = entry.getValue();
        ISpout spoutObject = (ISpout) taskData.getTaskObject();
        SpoutOutputCollectorImpl spoutOutputCollector = new SpoutOutputCollectorImpl(spoutObject, this, taskData, entry.getKey(), emittedCount, hasAckers, rand, hasEventLoggers, isDebug, pending);
        SpoutOutputCollector outputCollector = new SpoutOutputCollector(spoutOutputCollector);
        this.outputCollectors.add(outputCollector);
        taskData.getBuiltInMetrics().registerAll(stormConf, taskData.getUserContext());
        Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue);
        BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, taskData.getUserContext());
        if (spoutObject instanceof ICredentialsListener) {
            ((ICredentialsListener) spoutObject).setCredentials(credentials);
        }
        spoutObject.open(stormConf, taskData.getUserContext(), outputCollector);
    }
    openOrPrepareWasCalled.set(true);
    LOG.info("Opened spout {}:{}", componentId, idToTask.keySet());
    setupMetrics();
}