Examples with DisruptorQueue org.apache.storm.utils.DisruptorQueue used on opensource projects

Example 1 with DisruptorQueue

use of org.apache.storm.utils.DisruptorQueue in project storm by apache.

the class WorkerState method refreshLoad.

public void refreshLoad() {
    Set<Integer> remoteTasks = Sets.difference(new HashSet<Integer>(outboundTasks), new HashSet<>(taskIds));
    Long now = System.currentTimeMillis();
    Map<Integer, Double> localLoad = shortExecutorReceiveQueueMap.entrySet().stream().collect(Collectors.toMap((Function<Map.Entry<Integer, DisruptorQueue>, Integer>) Map.Entry::getKey, (Function<Map.Entry<Integer, DisruptorQueue>, Double>) entry -> {
        DisruptorQueue.QueueMetrics qMetrics = entry.getValue().getMetrics();
        return ((double) qMetrics.population()) / qMetrics.capacity();
    }));
    Map<Integer, Load> remoteLoad = new HashMap<>();
    cachedNodeToPortSocket.get().values().stream().forEach(conn -> remoteLoad.putAll(conn.getLoad(remoteTasks)));
    loadMapping.setLocal(localLoad);
    loadMapping.setRemote(remoteLoad);
    if (now > nextUpdate.get()) {
        receiver.sendLoadMetrics(localLoad);
        nextUpdate.set(now + LOAD_REFRESH_INTERVAL_MS);
    }
}

Example 2 with DisruptorQueue

use of org.apache.storm.utils.DisruptorQueue in project storm by apache.

the class WorkerState method transferLocal.

public void transferLocal(List<AddressedTuple> tupleBatch) {
    Map<Integer, List<AddressedTuple>> grouped = new HashMap<>();
    for (AddressedTuple tuple : tupleBatch) {
        Integer executor = taskToShortExecutor.get(tuple.dest);
        if (null == executor) {
            LOG.warn("Received invalid messages for unknown tasks. Dropping... ");
            continue;
        }
        List<AddressedTuple> current = grouped.get(executor);
        if (null == current) {
            current = new ArrayList<>();
            grouped.put(executor, current);
        }
        current.add(tuple);
    }
    for (Map.Entry<Integer, List<AddressedTuple>> entry : grouped.entrySet()) {
        DisruptorQueue queue = shortExecutorReceiveQueueMap.get(entry.getKey());
        if (null != queue) {
            queue.publish(entry.getValue());
        } else {
            LOG.warn("Received invalid messages for unknown tasks. Dropping... ");
        }
    }
}

Example 3 with DisruptorQueue

use of org.apache.storm.utils.DisruptorQueue in project storm by apache.

the class BoltExecutor method init.

public void init(Map<Integer, Task> idToTask) {
    while (!stormActive.get()) {
        Utils.sleep(100);
    }
    LOG.info("Preparing bolt {}:{}", componentId, idToTask.keySet());
    for (Map.Entry<Integer, Task> entry : idToTask.entrySet()) {
        Task taskData = entry.getValue();
        IBolt boltObject = (IBolt) taskData.getTaskObject();
        TopologyContext userContext = taskData.getUserContext();
        taskData.getBuiltInMetrics().registerAll(stormConf, userContext);
        if (boltObject instanceof ICredentialsListener) {
            ((ICredentialsListener) boltObject).setCredentials(credentials);
        }
        if (Constants.SYSTEM_COMPONENT_ID.equals(componentId)) {
            Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue, "transfer", workerData.getTransferQueue());
            BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, userContext);
            Map cachedNodePortToSocket = (Map) workerData.getCachedNodeToPortSocket().get();
            BuiltinMetricsUtil.registerIconnectionClientMetrics(cachedNodePortToSocket, stormConf, userContext);
            BuiltinMetricsUtil.registerIconnectionServerMetric(workerData.getReceiver(), stormConf, userContext);
        } else {
            Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue);
            BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, userContext);
        }
        IOutputCollector outputCollector = new BoltOutputCollectorImpl(this, taskData, entry.getKey(), rand, hasEventLoggers, isDebug);
        boltObject.prepare(stormConf, userContext, new OutputCollector(outputCollector));
    }
    openOrPrepareWasCalled.set(true);
    LOG.info("Prepared bolt {}:{}", componentId, idToTask.keySet());
    setupMetrics();
}

Example 4 with DisruptorQueue

use of org.apache.storm.utils.DisruptorQueue in project storm by apache.

the class SpoutExecutor method init.

public void init(final Map<Integer, Task> idToTask) {
    while (!stormActive.get()) {
        Utils.sleep(100);
    }
    LOG.info("Opening spout {}:{}", componentId, idToTask.keySet());
    this.idToTask = idToTask;
    this.maxSpoutPending = Utils.getInt(stormConf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), 0) * idToTask.size();
    this.spouts = new ArrayList<>();
    for (Task task : idToTask.values()) {
        this.spouts.add((ISpout) task.getTaskObject());
    }
    this.pending = new RotatingMap<>(2, new RotatingMap.ExpiredCallback<Long, TupleInfo>() {

        @Override
        public void expire(Long key, TupleInfo tupleInfo) {
            Long timeDelta = null;
            if (tupleInfo.getTimestamp() != 0) {
                timeDelta = Time.deltaMs(tupleInfo.getTimestamp());
            }
            failSpoutMsg(SpoutExecutor.this, idToTask.get(tupleInfo.getTaskId()), timeDelta, tupleInfo, "TIMEOUT");
        }
    });
    this.spoutThrottlingMetrics.registerAll(stormConf, idToTask.values().iterator().next().getUserContext());
    this.outputCollectors = new ArrayList<>();
    for (Map.Entry<Integer, Task> entry : idToTask.entrySet()) {
        Task taskData = entry.getValue();
        ISpout spoutObject = (ISpout) taskData.getTaskObject();
        SpoutOutputCollectorImpl spoutOutputCollector = new SpoutOutputCollectorImpl(spoutObject, this, taskData, entry.getKey(), emittedCount, hasAckers, rand, hasEventLoggers, isDebug, pending);
        SpoutOutputCollector outputCollector = new SpoutOutputCollector(spoutOutputCollector);
        this.outputCollectors.add(outputCollector);
        taskData.getBuiltInMetrics().registerAll(stormConf, taskData.getUserContext());
        Map<String, DisruptorQueue> map = ImmutableMap.of("sendqueue", transferQueue, "receive", receiveQueue);
        BuiltinMetricsUtil.registerQueueMetrics(map, stormConf, taskData.getUserContext());
        if (spoutObject instanceof ICredentialsListener) {
            ((ICredentialsListener) spoutObject).setCredentials(credentials);
        }
        spoutObject.open(stormConf, taskData.getUserContext(), outputCollector);
    }
    openOrPrepareWasCalled.set(true);
    LOG.info("Opened spout {}:{}", componentId, idToTask.keySet());
    setupMetrics();
}