Examples with AsyncLoopThread com.alibaba.jstorm.callback.AsyncLoopThread used on opensource projects

Example 1 with AsyncLoopThread

use of com.alibaba.jstorm.callback.AsyncLoopThread in project jstorm by alibaba.

the class NimbusServer method cleanup.

public void cleanup() {
    if (data == null || data.getIsShutdown().getAndSet(true)) {
        LOG.info("Notify to quit nimbus");
        return;
    }
    LOG.info("Begin to shutdown nimbus");
    AsyncLoopRunnable.getShutdown().set(true);
    data.getScheduExec().shutdownNow();
    for (AsyncLoopThread t : smartThreads) {
        t.cleanup();
        JStormUtils.sleepMs(10);
        t.interrupt();
        // try {
        // t.join();
        // } catch (InterruptedException e) {
        // LOG.error("join thread", e);
        // }
        LOG.info("Successfully cleanup " + t.getThread().getName());
    }
    if (serviceHandler != null) {
        serviceHandler.shutdown();
    }
    if (topologyAssign != null) {
        topologyAssign.cleanup();
        LOG.info("Successfully shutdown TopologyAssign thread");
    }
    if (follower != null) {
        follower.clean();
        LOG.info("Successfully shutdown follower thread");
    }
    if (data != null) {
        data.cleanup();
        LOG.info("Successfully shutdown NimbusData");
    }
    if (thriftServer != null) {
        thriftServer.stop();
        LOG.info("Successfully shutdown thrift server");
    }
    if (hs != null) {
        hs.shutdown();
        LOG.info("Successfully shutdown httpserver");
    }
    LOG.info("Successfully shutdown nimbus");
    // make sure shutdown nimbus
    JStormUtils.halt_process(0, "!!!Shutdown!!!");
}

Example 2 with AsyncLoopThread

use of com.alibaba.jstorm.callback.AsyncLoopThread in project jstorm by alibaba.

the class WorkerData method setSerializeThreads.

protected List<AsyncLoopThread> setSerializeThreads() {
    WorkerTopologyContext workerTopologyContext = contextMaker.makeWorkerTopologyContext(sysTopology);
    int tasksNum = shutdownTasks.size();
    double workerRatio = ConfigExtension.getWorkerSerializeThreadRatio(stormConf);
    int workerSerialThreadNum = Utils.getInt(Math.ceil(workerRatio * tasksNum));
    if (workerSerialThreadNum > 0 && tasksNum > 0) {
        double average = tasksNum / (double) workerSerialThreadNum;
        for (int i = 0; i < workerSerialThreadNum; i++) {
            int startRunTaskIndex = Utils.getInt(Math.rint(average * i));
            serializeThreads.add(new AsyncLoopThread(new WorkerSerializeRunnable(shutdownTasks, stormConf, workerTopologyContext, startRunTaskIndex, i)));
        }
    }
    return serializeThreads;
}

Example 3 with AsyncLoopThread

use of com.alibaba.jstorm.callback.AsyncLoopThread in project jstorm by alibaba.

the class Supervisor method mkSupervisor.

/**
 * create and start a supervisor
 *
 * @param conf          : configuration (default.yaml & storm.yaml)
 * @param sharedContext : null (right now)
 * @return SupervisorManger: which is used to shutdown all workers and supervisor
 */
@SuppressWarnings("rawtypes")
public SupervisorManger mkSupervisor(Map conf, IContext sharedContext) throws Exception {
    LOG.info("Starting Supervisor with conf " + conf);
    /**
     * Step 1: cleanup all files in /storm-local-dir/supervisor/tmp
     */
    String path = StormConfig.supervisorTmpDir(conf);
    FileUtils.cleanDirectory(new File(path));
    /**
     * Step 2: create ZK operation instance StormClusterState
     */
    StormClusterState stormClusterState = Cluster.mk_storm_cluster_state(conf);
    String hostName = JStormServerUtils.getHostName(conf);
    WorkerReportError workerReportError = new WorkerReportError(stormClusterState, hostName);
    /**
     * Step 3, create LocalStat (a simple KV store)
     * 3.1 create LocalState instance;
     * 3.2 get supervisorId, if there's no supervisorId, create one
     */
    LocalState localState = StormConfig.supervisorState(conf);
    String supervisorId = (String) localState.get(Common.LS_ID);
    if (supervisorId == null) {
        supervisorId = UUID.randomUUID().toString();
        localState.put(Common.LS_ID, supervisorId);
    }
    // clean LocalStat's zk-assignment & versions
    localState.remove(Common.LS_LOCAl_ZK_ASSIGNMENTS);
    localState.remove(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION);
    Vector<AsyncLoopThread> threads = new Vector<>();
    // Step 4 create HeartBeat
    // every supervisor.heartbeat.frequency.secs, write SupervisorInfo to ZK
    // sync heartbeat to nimbus
    Heartbeat hb = new Heartbeat(conf, stormClusterState, supervisorId, localState);
    hb.update();
    AsyncLoopThread heartbeat = new AsyncLoopThread(hb, false, null, Thread.MIN_PRIORITY, true);
    threads.add(heartbeat);
    // Sync heartbeat to Apsara Container
    AsyncLoopThread syncContainerHbThread = SyncContainerHb.mkSupervisorInstance(conf);
    if (syncContainerHbThread != null) {
        threads.add(syncContainerHbThread);
    }
    // Step 5 create and start sync Supervisor thread
    // every supervisor.monitor.frequency.secs second run SyncSupervisor
    ConcurrentHashMap<String, String> workerThreadPids = new ConcurrentHashMap<>();
    SyncProcessEvent syncProcessEvent = new SyncProcessEvent(supervisorId, conf, localState, workerThreadPids, sharedContext, workerReportError, stormClusterState);
    EventManagerImp syncSupEventManager = new EventManagerImp();
    AsyncLoopThread syncSupEventThread = new AsyncLoopThread(syncSupEventManager);
    threads.add(syncSupEventThread);
    SyncSupervisorEvent syncSupervisorEvent = new SyncSupervisorEvent(supervisorId, conf, syncSupEventManager, stormClusterState, localState, syncProcessEvent, hb);
    int syncFrequency = JStormUtils.parseInt(conf.get(Config.SUPERVISOR_MONITOR_FREQUENCY_SECS));
    EventManagerPusher syncSupervisorPusher = new EventManagerPusher(syncSupEventManager, syncSupervisorEvent, syncFrequency);
    AsyncLoopThread syncSupervisorThread = new AsyncLoopThread(syncSupervisorPusher);
    threads.add(syncSupervisorThread);
    // Step 6 start httpserver
    Httpserver httpserver = null;
    if (!StormConfig.local_mode(conf)) {
        int port = ConfigExtension.getSupervisorDeamonHttpserverPort(conf);
        httpserver = new Httpserver(port, conf);
        httpserver.start();
    }
    // Step 7 check supervisor
    if (!StormConfig.local_mode(conf)) {
        if (ConfigExtension.isEnableCheckSupervisor(conf)) {
            SupervisorHealth supervisorHealth = new SupervisorHealth(conf, hb, supervisorId);
            AsyncLoopThread healthThread = new AsyncLoopThread(supervisorHealth, false, null, Thread.MIN_PRIORITY, true);
            threads.add(healthThread);
        }
        // init refresh config thread
        AsyncLoopThread refreshConfigThread = new AsyncLoopThread(new SupervisorRefreshConfig(conf));
        threads.add(refreshConfigThread);
    }
    // create SupervisorManger which can shutdown all supervisor and workers
    return new SupervisorManger(conf, supervisorId, threads, syncSupEventManager, httpserver, stormClusterState, workerThreadPids);
}

Example 4 with AsyncLoopThread

use of com.alibaba.jstorm.callback.AsyncLoopThread in project jstorm by alibaba.

the class Worker method execute.

public WorkerShutdown execute() throws Exception {
    List<AsyncLoopThread> threads = new ArrayList<>();
    // create recv connection, reduce the count of netty client reconnect
    AsyncLoopThread controlRvthread = startDispatchThread();
    threads.add(controlRvthread);
    // create client before create task
    // so create client connection before create task
    // refresh connection
    RefreshConnections refreshConn = makeRefreshConnections();
    AsyncLoopThread refreshconn = new AsyncLoopThread(refreshConn, false, Thread.MIN_PRIORITY, true);
    threads.add(refreshconn);
    // refresh ZK active status
    RefreshActive refreshZkActive = new RefreshActive(workerData);
    AsyncLoopThread refreshZk = new AsyncLoopThread(refreshZkActive, false, Thread.MIN_PRIORITY, true);
    threads.add(refreshZk);
    // create send control message thread
    DrainerCtrlRunable drainerCtrlRunable;
    drainerCtrlRunable = new DrainerCtrlRunable(workerData, MetricDef.SEND_THREAD);
    AsyncLoopThread controlSendThread = new AsyncLoopThread(drainerCtrlRunable, false, Thread.MAX_PRIORITY, true);
    threads.add(controlSendThread);
    // Sync heartbeat to Apsara Container
    AsyncLoopThread syncContainerHbThread = SyncContainerHb.mkWorkerInstance(workerData.getStormConf());
    if (syncContainerHbThread != null) {
        threads.add(syncContainerHbThread);
    }
    JStormMetricsReporter metricReporter = new JStormMetricsReporter(workerData);
    metricReporter.init();
    workerData.setMetricsReporter(metricReporter);
    // refresh heartbeat to Local dir
    RunnableCallback heartbeatFn = new WorkerHeartbeatRunable(workerData);
    AsyncLoopThread hb = new AsyncLoopThread(heartbeatFn, false, null, Thread.NORM_PRIORITY, true);
    threads.add(hb);
    // shutdown task callbacks
    List<TaskShutdownDameon> shutdownTasks = createTasks();
    workerData.setShutdownTasks(shutdownTasks);
    // create worker serializes/deserialize threads
    List<AsyncLoopThread> serializeThreads = workerData.setSerializeThreads();
    threads.addAll(serializeThreads);
    List<AsyncLoopThread> deserializeThreads = workerData.setDeserializeThreads();
    threads.addAll(deserializeThreads);
    return new WorkerShutdown(workerData, threads);
}

Example 5 with AsyncLoopThread

use of com.alibaba.jstorm.callback.AsyncLoopThread in project jstorm by alibaba.

the class WorkerShutdown method shutdown.

@Override
public void shutdown() {
    if (shutdown.getAndSet(true)) {
        LOG.info("Worker has been shutdown already");
        return;
    }
    // dump worker jstack, jmap info to specific file
    if (ConfigExtension.isOutworkerDump(conf))
        workerDumpInfoOutput();
    // shutdown tasks
    List<Future<?>> futures = new ArrayList<>();
    for (ShutdownableDameon task : shutdownTasks) {
        Future<?> future = flusherPool.submit(task);
        futures.add(future);
    }
    // To be assure all tasks are closed rightly
    JStormServerUtils.checkFutures(futures);
    if (recvConnection != null) {
        recvConnection.close();
    }
    AsyncLoopRunnable.getShutdown().set(true);
    threadPool.shutdown();
    flusherPool.shutdown();
    try {
        flusherPool.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        LOG.error("Failed to shutdown client scheduler", e);
    }
    // refreshconn, refreshzk, hb, drainer
    for (AsyncLoopThread t : threads) {
        LOG.info("Begin to shutdown " + t.getThread().getName());
        t.cleanup();
        JStormUtils.sleepMs(100);
        t.interrupt();
        // try {
        // t.join();
        // } catch (InterruptedException e) {
        // LOG.error("join thread", e);
        // }
        LOG.info("Successfully " + t.getThread().getName());
    }
    // send data to close connection
    for (WorkerSlot k : nodePortToSocket.keySet()) {
        IConnection value = nodePortToSocket.get(k);
        value.close();
    }
    context.term();
    // close ZK client
    try {
        zkCluster.disconnect();
        cluster_state.close();
    } catch (Exception e) {
        LOG.info("Shutdown error,", e);
    }
    String clusterMode = StormConfig.cluster_mode(conf);
    if (clusterMode.equals("distributed")) {
        // Only halt process in distributed mode. Because the worker is a fake process in local mode.
        JStormUtils.halt_process(0, "!!!Shutdown!!!");
    }
}