Search in sources :

Example 6 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class Client method send.

@Override
public void send(int taskId, byte[] payload) {
    TaskMessage msg = new TaskMessage(taskId, payload);
    List<TaskMessage> wrapper = new ArrayList<TaskMessage>(1);
    wrapper.add(msg);
    send(wrapper.iterator());
}
Also used : ArrayList(java.util.ArrayList) TaskMessage(org.apache.storm.messaging.TaskMessage)

Example 7 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class Server method sendLoadMetrics.

@Override
public void sendLoadMetrics(Map<Integer, Double> taskToLoad) {
    try {
        MessageBatch mb = new MessageBatch(1);
        mb.add(new TaskMessage(-1, _ser.serialize(Arrays.asList((Object) taskToLoad))));
        allChannels.write(mb);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
Also used : IStatefulObject(org.apache.storm.metric.api.IStatefulObject) IOException(java.io.IOException) TaskMessage(org.apache.storm.messaging.TaskMessage)

Example 8 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class Worker method start.

public void start() throws Exception {
    LOG.info("Launching worker for {} on {}:{} with id {} and conf {}", topologyId, assignmentId, port, workerId, conf);
    // if ConfigUtils.isLocalMode(conf) returns false then it is in distributed mode.
    if (!ConfigUtils.isLocalMode(conf)) {
        // Distributed mode
        SysOutOverSLF4J.sendSystemOutAndErrToSLF4J();
        String pid = Utils.processPid();
        FileUtils.touch(new File(ConfigUtils.workerPidPath(conf, workerId, pid)));
        FileUtils.writeStringToFile(new File(ConfigUtils.workerArtifactsPidPath(conf, topologyId, port)), pid, Charset.forName("UTF-8"));
    }
    final Map topologyConf = ConfigUtils.overrideLoginConfigWithSystemProperty(ConfigUtils.readSupervisorStormConf(conf, topologyId));
    List<ACL> acls = Utils.getWorkerACL(topologyConf);
    IStateStorage stateStorage = ClusterUtils.mkStateStorage(conf, topologyConf, acls, new ClusterStateContext(DaemonType.WORKER));
    IStormClusterState stormClusterState = ClusterUtils.mkStormClusterState(stateStorage, acls, new ClusterStateContext());
    Credentials initialCredentials = stormClusterState.credentials(topologyId, null);
    Map<String, String> initCreds = new HashMap<>();
    if (initialCredentials != null) {
        initCreds.putAll(initialCredentials.get_creds());
    }
    autoCreds = AuthUtils.GetAutoCredentials(topologyConf);
    subject = AuthUtils.populateSubject(null, autoCreds, initCreds);
    Subject.doAs(subject, new PrivilegedExceptionAction<Object>() {

        @Override
        public Object run() throws Exception {
            workerState = new WorkerState(conf, context, topologyId, assignmentId, port, workerId, topologyConf, stateStorage, stormClusterState);
            // Heartbeat here so that worker process dies if this fails
            // it's important that worker heartbeat to supervisor ASAP so that supervisor knows
            // that worker is running and moves on
            doHeartBeat();
            executorsAtom = new AtomicReference<>(null);
            // launch heartbeat threads immediately so that slow-loading tasks don't cause the worker to timeout
            // to the supervisor
            workerState.heartbeatTimer.scheduleRecurring(0, (Integer) conf.get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS), () -> {
                try {
                    doHeartBeat();
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            });
            workerState.executorHeartbeatTimer.scheduleRecurring(0, (Integer) conf.get(Config.WORKER_HEARTBEAT_FREQUENCY_SECS), Worker.this::doExecutorHeartbeats);
            workerState.registerCallbacks();
            workerState.refreshConnections(null);
            workerState.activateWorkerWhenAllConnectionsReady();
            workerState.refreshStormActive(null);
            workerState.runWorkerStartHooks();
            List<IRunningExecutor> newExecutors = new ArrayList<IRunningExecutor>();
            for (List<Long> e : workerState.getExecutors()) {
                if (ConfigUtils.isLocalMode(topologyConf)) {
                    newExecutors.add(LocalExecutor.mkExecutor(workerState, e, initCreds).execute());
                } else {
                    newExecutors.add(Executor.mkExecutor(workerState, e, initCreds).execute());
                }
            }
            executorsAtom.set(newExecutors);
            EventHandler<Object> tupleHandler = (packets, seqId, batchEnd) -> workerState.sendTuplesToRemoteWorker((HashMap<Integer, ArrayList<TaskMessage>>) packets, seqId, batchEnd);
            // This thread will publish the messages destined for remote tasks to remote connections
            transferThread = Utils.asyncLoop(() -> {
                workerState.transferQueue.consumeBatchWhenAvailable(tupleHandler);
                return 0L;
            });
            DisruptorBackpressureCallback disruptorBackpressureHandler = mkDisruptorBackpressureHandler(workerState);
            workerState.transferQueue.registerBackpressureCallback(disruptorBackpressureHandler);
            workerState.transferQueue.setEnableBackpressure((Boolean) topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE));
            workerState.transferQueue.setHighWaterMark(Utils.getDouble(topologyConf.get(Config.BACKPRESSURE_DISRUPTOR_HIGH_WATERMARK)));
            workerState.transferQueue.setLowWaterMark(Utils.getDouble(topologyConf.get(Config.BACKPRESSURE_DISRUPTOR_LOW_WATERMARK)));
            WorkerBackpressureCallback backpressureCallback = mkBackpressureHandler();
            backpressureThread = new WorkerBackpressureThread(workerState.backpressureTrigger, workerState, backpressureCallback);
            if ((Boolean) topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE)) {
                backpressureThread.start();
                stormClusterState.topologyBackpressure(topologyId, workerState::refreshThrottle);
                int pollingSecs = Utils.getInt(topologyConf.get(Config.TASK_BACKPRESSURE_POLL_SECS));
                workerState.refreshBackpressureTimer.scheduleRecurring(0, pollingSecs, workerState::refreshThrottle);
            }
            credentialsAtom = new AtomicReference<Credentials>(initialCredentials);
            establishLogSettingCallback();
            workerState.stormClusterState.credentials(topologyId, Worker.this::checkCredentialsChanged);
            workerState.refreshCredentialsTimer.scheduleRecurring(0, (Integer) conf.get(Config.TASK_CREDENTIALS_POLL_SECS), new Runnable() {

                @Override
                public void run() {
                    checkCredentialsChanged();
                    if ((Boolean) topologyConf.get(Config.TOPOLOGY_BACKPRESSURE_ENABLE)) {
                        checkThrottleChanged();
                    }
                }
            });
            // The jitter allows the clients to get the data at different times, and avoids thundering herd
            if (!(Boolean) topologyConf.get(Config.TOPOLOGY_DISABLE_LOADAWARE_MESSAGING)) {
                workerState.refreshLoadTimer.scheduleRecurringWithJitter(0, 1, 500, workerState::refreshLoad);
            }
            workerState.refreshConnectionsTimer.scheduleRecurring(0, (Integer) conf.get(Config.TASK_REFRESH_POLL_SECS), workerState::refreshConnections);
            workerState.resetLogLevelsTimer.scheduleRecurring(0, (Integer) conf.get(Config.WORKER_LOG_LEVEL_RESET_POLL_SECS), logConfigManager::resetLogLevels);
            workerState.refreshActiveTimer.scheduleRecurring(0, (Integer) conf.get(Config.TASK_REFRESH_POLL_SECS), workerState::refreshStormActive);
            LOG.info("Worker has topology config {}", Utils.redactValue(topologyConf, Config.STORM_ZOOKEEPER_TOPOLOGY_AUTH_PAYLOAD));
            LOG.info("Worker {} for storm {} on {}:{}  has finished loading", workerId, topologyId, assignmentId, port);
            return this;
        }

        ;
    });
}
Also used : WorkerBackpressureCallback(org.apache.storm.utils.WorkerBackpressureCallback) HashMap(java.util.HashMap) EventHandler(com.lmax.disruptor.EventHandler) IRunningExecutor(org.apache.storm.executor.IRunningExecutor) List(java.util.List) ArrayList(java.util.ArrayList) IStormClusterState(org.apache.storm.cluster.IStormClusterState) DisruptorBackpressureCallback(org.apache.storm.utils.DisruptorBackpressureCallback) ACL(org.apache.zookeeper.data.ACL) AtomicReference(java.util.concurrent.atomic.AtomicReference) IOException(java.io.IOException) IOException(java.io.IOException) WorkerBackpressureThread(org.apache.storm.utils.WorkerBackpressureThread) File(java.io.File) Map(java.util.Map) HashMap(java.util.HashMap) Credentials(org.apache.storm.generated.Credentials) IAutoCredentials(org.apache.storm.security.auth.IAutoCredentials) IStateStorage(org.apache.storm.cluster.IStateStorage) ClusterStateContext(org.apache.storm.cluster.ClusterStateContext) TaskMessage(org.apache.storm.messaging.TaskMessage)

Example 9 with TaskMessage

use of org.apache.storm.messaging.TaskMessage in project storm by apache.

the class TransferDrainer method send.

public void send(Map<Integer, NodeInfo> taskToNode, Map<NodeInfo, IConnection> connections) {
    HashMap<NodeInfo, ArrayList<ArrayList<TaskMessage>>> bundleMapByDestination = groupBundleByDestination(taskToNode);
    for (Map.Entry<NodeInfo, ArrayList<ArrayList<TaskMessage>>> entry : bundleMapByDestination.entrySet()) {
        NodeInfo hostPort = entry.getKey();
        IConnection connection = connections.get(hostPort);
        if (null != connection) {
            ArrayList<ArrayList<TaskMessage>> bundle = entry.getValue();
            Iterator<TaskMessage> iter = getBundleIterator(bundle);
            if (null != iter && iter.hasNext()) {
                connection.send(iter);
            }
        } else {
            LOG.warn("Connection is not available for hostPort {}", hostPort);
        }
    }
}
Also used : NodeInfo(org.apache.storm.generated.NodeInfo) ArrayList(java.util.ArrayList) IConnection(org.apache.storm.messaging.IConnection) Map(java.util.Map) HashMap(java.util.HashMap) TaskMessage(org.apache.storm.messaging.TaskMessage)

Aggregations

TaskMessage (org.apache.storm.messaging.TaskMessage)9 ArrayList (java.util.ArrayList)4 IOException (java.io.IOException)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 EventHandler (com.lmax.disruptor.EventHandler)1 File (java.io.File)1 Iterator (java.util.Iterator)1 List (java.util.List)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 ClusterStateContext (org.apache.storm.cluster.ClusterStateContext)1 IStateStorage (org.apache.storm.cluster.IStateStorage)1 IStormClusterState (org.apache.storm.cluster.IStormClusterState)1 IRunningExecutor (org.apache.storm.executor.IRunningExecutor)1 Credentials (org.apache.storm.generated.Credentials)1 NodeInfo (org.apache.storm.generated.NodeInfo)1 IConnection (org.apache.storm.messaging.IConnection)1 IStatefulObject (org.apache.storm.metric.api.IStatefulObject)1 IAutoCredentials (org.apache.storm.security.auth.IAutoCredentials)1 AddressedTuple (org.apache.storm.tuple.AddressedTuple)1