Examples with RunnableCallback com.alibaba.jstorm.callback.RunnableCallback used on opensource projects

Example 1 with RunnableCallback

use of com.alibaba.jstorm.callback.RunnableCallback in project jstorm by alibaba.

the class Worker method execute.

public WorkerShutdown execute() throws Exception {
    List<AsyncLoopThread> threads = new ArrayList<>();
    // create recv connection, reduce the count of netty client reconnect
    AsyncLoopThread controlRvthread = startDispatchThread();
    threads.add(controlRvthread);
    // create client before create task
    // so create client connection before create task
    // refresh connection
    RefreshConnections refreshConn = makeRefreshConnections();
    AsyncLoopThread refreshconn = new AsyncLoopThread(refreshConn, false, Thread.MIN_PRIORITY, true);
    threads.add(refreshconn);
    // refresh ZK active status
    RefreshActive refreshZkActive = new RefreshActive(workerData);
    AsyncLoopThread refreshzk = new AsyncLoopThread(refreshZkActive, false, Thread.MIN_PRIORITY, true);
    threads.add(refreshzk);
    //create send control message thread
    DrainerCtrlRunable drainerCtrlRunable;
    drainerCtrlRunable = new DrainerCtrlRunable(workerData, MetricDef.SEND_THREAD);
    AsyncLoopThread controlSendThread = new AsyncLoopThread(drainerCtrlRunable, false, Thread.MAX_PRIORITY, true);
    threads.add(controlSendThread);
    // Sync heartbeat to Apsara Container
    AsyncLoopThread syncContainerHbThread = SyncContainerHb.mkWorkerInstance(workerData.getStormConf());
    if (syncContainerHbThread != null) {
        threads.add(syncContainerHbThread);
    }
    JStormMetricsReporter metricReporter = new JStormMetricsReporter(workerData);
    metricReporter.init();
    workerData.setMetricsReporter(metricReporter);
    // refresh hearbeat to Local dir
    RunnableCallback heartbeatFn = new WorkerHeartbeatRunable(workerData);
    AsyncLoopThread hb = new AsyncLoopThread(heartbeatFn, false, null, Thread.NORM_PRIORITY, true);
    threads.add(hb);
    // shutdown task callbacks
    List<TaskShutdownDameon> shutdownTasks = createTasks();
    workerData.setShutdownTasks(shutdownTasks);
    //create worker serializes/deserializes
    List<AsyncLoopThread> serializeThreads = workerData.setSerializeThreads();
    threads.addAll(serializeThreads);
    List<AsyncLoopThread> deserializeThreads = workerData.setDeserializeThreads();
    threads.addAll(deserializeThreads);
    return new WorkerShutdown(workerData, threads);
}

Example 2 with RunnableCallback

use of com.alibaba.jstorm.callback.RunnableCallback in project jstorm by alibaba.

the class FollowerRunnable method tryToBeLeader.

private void tryToBeLeader(final Map conf) throws Exception {
    boolean allowed = check_nimbus_priority();
    if (allowed) {
        RunnableCallback masterCallback = new RunnableCallback() {

            @Override
            public void run() {
                try {
                    tryToBeLeader(conf);
                } catch (Exception e) {
                    LOG.error("To be master error", e);
                    JStormUtils.halt_process(30, "Cant't to be master" + e.getMessage());
                }
            }
        };
        LOG.info("This nimbus can be  leader");
        data.getStormClusterState().try_to_be_leader(Cluster.MASTER_SUBTREE, hostPort, masterCallback);
    } else {
        LOG.info("This nimbus can't be leader");
    }
}

Example 3 with RunnableCallback

use of com.alibaba.jstorm.callback.RunnableCallback in project jstorm by alibaba.

the class Task method execute.

public TaskShutdownDameon execute() throws Exception {
    taskSendTargets = echoToSystemBolt();
    // create thread to get tuple from zeroMQ,
    // and pass the tuple to bolt/spout
    taskTransfer = mkTaskSending(workerData);
    RunnableCallback baseExecutor = prepareExecutor();
    //set baseExecutors for update
    setBaseExecutors((BaseExecutors) baseExecutor);
    AsyncLoopThread executor_threads = new AsyncLoopThread(baseExecutor, false, Thread.MAX_PRIORITY, true);
    taskReceiver = mkTaskReceiver();
    List<AsyncLoopThread> allThreads = new ArrayList<AsyncLoopThread>();
    allThreads.add(executor_threads);
    LOG.info("Finished loading task " + componentId + ":" + taskId);
    taskShutdownDameon = getShutdown(allThreads, baseExecutor);
    return taskShutdownDameon;
}

Example 4 with RunnableCallback

use of com.alibaba.jstorm.callback.RunnableCallback in project jstorm by alibaba.

the class NimbusServer method mkRefreshConfThread.

/**
     * handle manual conf changes, check every 15 sec
     */
private void mkRefreshConfThread(final NimbusData nimbusData) {
    nimbusData.getScheduExec().scheduleAtFixedRate(new RunnableCallback() {

        @Override
        public void run() {
            LOG.debug("checking changes in storm.yaml...");
            Map newConf = Utils.readStormConfig();
            if (Utils.isConfigChanged(nimbusData.getConf(), newConf)) {
                LOG.warn("detected changes in storm.yaml, updating...");
                synchronized (nimbusData.getConf()) {
                    nimbusData.getConf().clear();
                    nimbusData.getConf().putAll(newConf);
                }
                RefreshableComponents.refresh(newConf);
            } else {
                LOG.debug("no changes detected, stay put.");
            }
        }

        @Override
        public Object getResult() {
            return 15;
        }
    }, 15, 15, TimeUnit.SECONDS);
    LOG.info("Successfully init configuration refresh thread");
}

Example 5 with RunnableCallback

use of com.alibaba.jstorm.callback.RunnableCallback in project jstorm by alibaba.

the class SyncSupervisorEvent method run.

@Override
public void run() {
    LOG.debug("Synchronizing supervisor, interval seconds:" + TimeUtils.time_delta(lastTime));
    lastTime = TimeUtils.current_time_secs();
    //In order to ensure that the status is the same for each execution of syncsupervisor
    MachineCheckStatus checkStatus = new MachineCheckStatus();
    checkStatus.SetType(heartbeat.getCheckStatus().getType());
    try {
        RunnableCallback syncCallback = new EventManagerZkPusher(this, syncSupEventManager);
        Map<String, Integer> assignmentVersion = (Map<String, Integer>) localState.get(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION);
        if (assignmentVersion == null) {
            assignmentVersion = new HashMap<String, Integer>();
        }
        Map<String, Assignment> assignments = (Map<String, Assignment>) localState.get(Common.LS_LOCAl_ZK_ASSIGNMENTS);
        if (assignments == null) {
            assignments = new HashMap<String, Assignment>();
        }
        LOG.debug("get local assignments  " + assignments);
        LOG.debug("get local assignments version " + assignmentVersion);
        if (checkStatus.getType().equals(MachineCheckStatus.StatusType.panic) || checkStatus.getType().equals(MachineCheckStatus.StatusType.error)) {
            // if statuts is pannic or error, it will clear all assignments and kill all workers;
            assignmentVersion.clear();
            assignments.clear();
            LOG.warn("Supervisor Machine Check Status :" + checkStatus.getType() + ", so kill all workers.");
        } else {
            getAllAssignments(assignmentVersion, assignments, syncCallback);
        }
        LOG.debug("Get all assignments " + assignments);
        /**
             * Step 2: get topologyIds list from STORM-LOCAL-DIR/supervisor/stormdist/
             */
        List<String> downloadedTopologyIds = StormConfig.get_supervisor_toplogy_list(conf);
        LOG.debug("Downloaded storm ids: " + downloadedTopologyIds);
        /**
             * Step 3: get <port,LocalAssignments> from ZK local node's assignment
             */
        Map<Integer, LocalAssignment> zkAssignment;
        zkAssignment = getLocalAssign(stormClusterState, supervisorId, assignments);
        Map<Integer, LocalAssignment> localAssignment;
        /**
             * Step 4: writer local assignment to LocalState
             */
        try {
            LOG.debug("Writing local assignment " + zkAssignment);
            localAssignment = (Map<Integer, LocalAssignment>) localState.get(Common.LS_LOCAL_ASSIGNMENTS);
            if (localAssignment == null) {
                localAssignment = new HashMap<Integer, LocalAssignment>();
            }
            localState.put(Common.LS_LOCAL_ASSIGNMENTS, zkAssignment);
        } catch (IOException e) {
            LOG.error("put LS_LOCAL_ASSIGNMENTS " + zkAssignment + " of localState failed");
            throw e;
        }
        /**
             * Step 5: get reloaded topologys
             */
        Set<String> updateTopologys;
        updateTopologys = getUpdateTopologys(localAssignment, zkAssignment, assignments);
        Set<String> reDownloadTopologys = getNeedReDownloadTopologys(localAssignment);
        if (reDownloadTopologys != null) {
            updateTopologys.addAll(reDownloadTopologys);
        }
        /**
             * Step 6: download code from ZK
             */
        Map<String, String> topologyCodes = getTopologyCodeLocations(assignments, supervisorId);
        // downloadFailedTopologyIds which can't finished download binary from nimbus
        Set<String> downloadFailedTopologyIds = new HashSet<String>();
        downloadTopology(topologyCodes, downloadedTopologyIds, updateTopologys, assignments, downloadFailedTopologyIds);
        /**
             * Step 7: remove any downloaded useless topology
             */
        removeUselessTopology(topologyCodes, downloadedTopologyIds);
        /**
             * Step 7: push syncProcesses Event
             */
        // processEventManager.add(syncProcesses);
        syncProcesses.run(zkAssignment, downloadFailedTopologyIds);
        // If everything is OK, set the trigger to update heartbeat of
        // supervisor
        heartbeat.updateHbTrigger(true);
        try {
            // update localState
            localState.put(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION, assignmentVersion);
            localState.put(Common.LS_LOCAl_ZK_ASSIGNMENTS, assignments);
        } catch (IOException e) {
            LOG.error("put LS_LOCAL_ZK_ASSIGNMENT_VERSION&&LS_LOCAl_ZK_ASSIGNMENTS  failed");
            throw e;
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
    // throw new RuntimeException(e);
    }
    if (checkStatus.getType().equals(MachineCheckStatus.StatusType.panic)) {
        // if statuts is pannic, it will kill supervisor;
        JStormUtils.halt_process(0, "Supervisor Machine Check Status : Panic , !!!!shutdown!!!!");
    }
}