Examples with LocalState backtype.storm.utils.LocalState used on opensource projects

Example 1 with LocalState

use of backtype.storm.utils.LocalState in project storm-mesos by nathanmarz.

the class MesosSupervisor method prepare.

@Override
public void prepare(Map conf, String localDir) {
    try {
        _state = new LocalState(localDir);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    Semaphore initter = new Semaphore(0);
    _executor = new StormExecutor(initter);
    _driver = new MesosExecutorDriver(_executor);
    _driver.start();
    LOG.info("Waiting for executor to initialize...");
    try {
        initter.acquire();
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
    LOG.info("Executor initialized...");
    Thread suicide = new SuicideDetector(conf);
    suicide.setDaemon(true);
    suicide.start();
}

Example 2 with LocalState

use of backtype.storm.utils.LocalState in project storm-mesos by nathanmarz.

the class MesosNimbus method prepare.

@Override
public void prepare(Map conf, String localDir) {
    try {
        _conf = conf;
        _state = new LocalState(localDir);
        String id = (String) _state.get(FRAMEWORK_ID);
        _allowedHosts = listIntoSet((List) _conf.get(CONF_MESOS_ALLOWED_HOSTS));
        _disallowedHosts = listIntoSet((List) _conf.get(CONF_MESOS_DISALLOWED_HOSTS));
        Semaphore initter = new Semaphore(0);
        _scheduler = new NimbusScheduler(initter);
        Number failoverTimeout = (Number) conf.get(CONF_MASTER_FAILOVER_TIMEOUT_SECS);
        if (failoverTimeout == null)
            failoverTimeout = 3600;
        FrameworkInfo.Builder finfo = FrameworkInfo.newBuilder().setName("Storm-0.8.0").setFailoverTimeout(failoverTimeout.doubleValue()).setUser("");
        if (id != null) {
            finfo.setId(FrameworkID.newBuilder().setValue(id).build());
        }
        MesosSchedulerDriver driver = new MesosSchedulerDriver(_scheduler, finfo.build(), (String) conf.get(CONF_MASTER_URL));
        driver.start();
        LOG.info("Waiting for scheduler to initialize...");
        initter.acquire();
        LOG.info("Scheduler initialized...");
    } catch (IOException e) {
        throw new RuntimeException(e);
    } catch (InterruptedException e) {
        throw new RuntimeException(e);
    }
}

Example 3 with LocalState

use of backtype.storm.utils.LocalState in project jstorm by alibaba.

the class StormConfig method worker_state.

public static LocalState worker_state(Map conf, String id) throws IOException {
    String path = worker_heartbeats_root(conf, id);
    LocalState rtn = new LocalState(path);
    return rtn;
}

Example 4 with LocalState

use of backtype.storm.utils.LocalState in project jstorm by alibaba.

the class SyncProcessEvent method getLocalWorkerStats.

/**
 * get localStat of approved workerId's map
 *
 * @return Map[workerId, [worker heartbeat, state]]
 */
public Map<String, StateHeartbeat> getLocalWorkerStats(Map conf, LocalState localState, Map<Integer, LocalAssignment> assignedTasks) throws Exception {
    Map<String, StateHeartbeat> workerIdHbState = new HashMap<>();
    int now = TimeUtils.current_time_secs();
    /**
     * Get Map<workerId, WorkerHeartbeat> from local_dir/worker/ids/heartbeat
     */
    Map<String, WorkerHeartbeat> idToHeartbeat = readWorkerHeartbeats(conf);
    for (Map.Entry<String, WorkerHeartbeat> entry : idToHeartbeat.entrySet()) {
        String workerId = entry.getKey();
        WorkerHeartbeat whb = entry.getValue();
        State state;
        if (whb == null) {
            state = State.notStarted;
            Pair<Integer, Integer> timeToPort = this.workerIdToStartTimeAndPort.get(workerId);
            if (timeToPort != null) {
                LocalAssignment localAssignment = assignedTasks.get(timeToPort.getSecond());
                if (localAssignment == null) {
                    LOG.info("Following worker doesn't exist in the assignment, remove port=" + timeToPort.getSecond());
                    state = State.disallowed;
                    // workerId is disallowed ,so remove it from workerIdToStartTimeAndPort
                    Integer port = this.workerIdToStartTimeAndPort.get(workerId).getSecond();
                    this.workerIdToStartTimeAndPort.remove(workerId);
                    this.portToWorkerId.remove(port);
                }
            }
        } else if (!matchesAssignment(whb, assignedTasks)) {
            // workerId isn't approved or isn't assigned task
            state = State.disallowed;
        } else if ((now - whb.getTimeSecs()) > JStormUtils.parseInt(conf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS))) {
            if (!killingWorkers.containsKey(workerId)) {
                String outTimeInfo = " it is likely to be out of memory, the worker heartbeat has timed out ";
                workerReportError.report(whb.getTopologyId(), whb.getPort(), whb.getTaskIds(), outTimeInfo, ErrorConstants.CODE_WORKER_TIMEOUT);
            }
            state = State.timedOut;
        } else {
            if (isWorkerDead(workerId)) {
                if (!killingWorkers.containsKey(workerId)) {
                    String workerDeadInfo = "Worker is dead ";
                    workerReportError.report(whb.getTopologyId(), whb.getPort(), whb.getTaskIds(), workerDeadInfo, ErrorConstants.CODE_WORKER_DEAD);
                }
                state = State.timedOut;
            } else {
                state = State.valid;
            }
        }
        if (state != State.valid) {
            if (!killingWorkers.containsKey(workerId))
                LOG.info("Worker:" + workerId + " state:" + state + " WorkerHeartbeat:" + whb + " assignedTasks:" + assignedTasks + " at supervisor time-secs " + now);
        } else {
            LOG.debug("Worker:" + workerId + " state:" + state + " WorkerHeartbeat: " + whb + " at supervisor time-secs " + now);
        }
        workerIdHbState.put(workerId, new StateHeartbeat(state, whb));
    }
    return workerIdHbState;
}

Example 5 with LocalState

use of backtype.storm.utils.LocalState in project jstorm by alibaba.

the class SyncProcessEvent method checkNewWorkers.

/**
 * check whether timestamp of new workers is not > SUPERVISOR_WORKER_START_TIMEOUT_SECS, otherwise mark as failed
 */
public void checkNewWorkers(Map conf) throws IOException, InterruptedException {
    Set<String> workers = new HashSet<>();
    for (Entry<String, Pair<Integer, Integer>> entry : workerIdToStartTimeAndPort.entrySet()) {
        String workerId = entry.getKey();
        int startTime = entry.getValue().getFirst();
        LocalState ls = StormConfig.worker_state(conf, workerId);
        WorkerHeartbeat whb = (WorkerHeartbeat) ls.get(Common.LS_WORKER_HEARTBEAT);
        if (whb == null) {
            if ((TimeUtils.current_time_secs() - startTime) < JStormUtils.parseInt(conf.get(Config.SUPERVISOR_WORKER_START_TIMEOUT_SECS))) {
                LOG.info(workerId + " still hasn't started");
            } else {
                LOG.error("Failed to start Worker " + workerId);
                workers.add(workerId);
            }
        } else {
            LOG.info("Successfully started worker " + workerId);
            workers.add(workerId);
        }
    }
    for (String workerId : workers) {
        Integer port = this.workerIdToStartTimeAndPort.get(workerId).getSecond();
        this.workerIdToStartTimeAndPort.remove(workerId);
        this.portToWorkerId.remove(port);
    }
}