Examples with SupervisorInfo com.alibaba.jstorm.daemon.supervisor.SupervisorInfo used on opensource projects

Example 1 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class Cluster method get_all_SupervisorInfo.

/**
     * get all SupervisorInfo of storm cluster
     *
     * @param stormClusterState
     * @param callback
     * @return Map<String, SupervisorInfo> String: supervisorId SupervisorInfo: [time-secs hostname worker-ports uptime-secs]
     * @throws Exception
     */
public static Map<String, SupervisorInfo> get_all_SupervisorInfo(StormClusterState stormClusterState, RunnableCallback callback) throws Exception {
    Map<String, SupervisorInfo> rtn = new TreeMap<String, SupervisorInfo>();
    // get /ZK/supervisors
    List<String> supervisorIds = stormClusterState.supervisors(callback);
    // ignore any supervisors in blacklist
    List<String> blacklist = stormClusterState.get_blacklist();
    if (supervisorIds != null) {
        for (Iterator<String> iter = supervisorIds.iterator(); iter.hasNext(); ) {
            String supervisorId = iter.next();
            // get /supervisors/supervisorid
            SupervisorInfo supervisorInfo = stormClusterState.supervisor_info(supervisorId);
            if (supervisorInfo == null) {
                LOG.warn("Failed to get SupervisorInfo of " + supervisorId);
            } else if (blacklist.contains(supervisorInfo.getHostName())) {
                LOG.warn(" hostname:" + supervisorInfo.getHostName() + " is in blacklist");
            } else {
                rtn.put(supervisorId, supervisorInfo);
            }
        }
    } else {
        LOG.info("No alive supervisor");
    }
    return rtn;
}

Example 2 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class ServiceHandler method getSupervisorConf.

@Override
public String getSupervisorConf(String id) throws TException {
    Map<Object, Object> ret = new HashMap<>();
    ret.putAll(data.getConf());
    try {
        SupervisorInfo supervisorInfo = (SupervisorInfo) (((StormZkClusterState) this.data.getStormClusterState()).getObject(Cluster.supervisor_path(id), false));
        if (supervisorInfo != null && supervisorInfo.getSupervisorConf() != null) {
            ret.putAll(supervisorInfo.getSupervisorConf());
        } else {
            LOG.warn("supervisor conf not found in nimbus cache, supervisor id:{}, so fall back to nimbus conf", id);
        }
    } catch (Exception ex) {
        LOG.error("Error:", ex);
    }
    return JStormUtils.to_json(ret);
}

Example 3 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class ServiceHandler method getSupervisorWorkersByHostOrId.

/**
     * gets supervisor workers by host or supervisor id, note that id priors to host.
     *
     * @param host host
     * @param id   supervisor id
     * @return supervisor workers
     */
private SupervisorWorkers getSupervisorWorkersByHostOrId(String host, String id) throws TException {
    long start = System.nanoTime();
    if (StringUtils.isBlank(id) && StringUtils.isBlank(host)) {
        throw new TException("Must specify host or supervisor id!");
    }
    try {
        StormClusterState stormClusterState = data.getStormClusterState();
        // all supervisors
        Map<String, SupervisorInfo> supervisorInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
        SupervisorInfo supervisorInfo = null;
        String ip;
        if (!StringUtils.isBlank(id)) {
            supervisorInfo = supervisorInfos.get(id);
            host = supervisorInfo.getHostName();
            ip = NetWorkUtils.host2Ip(host);
        } else {
            ip = NetWorkUtils.host2Ip(host);
            for (Entry<String, SupervisorInfo> entry : supervisorInfos.entrySet()) {
                SupervisorInfo info = entry.getValue();
                if (info.getHostName().equals(host) || info.getHostName().equals(ip)) {
                    id = entry.getKey();
                    supervisorInfo = info;
                    break;
                }
            }
        }
        if (supervisorInfo == null) {
            throw new TException("unknown supervisor id:" + id);
        }
        Map<String, Assignment> assignments = Cluster.get_all_assignment(stormClusterState, null);
        Map<Integer, WorkerSummary> portWorkerSummarys = new TreeMap<Integer, WorkerSummary>();
        int usedSlotNumber = 0;
        Map<String, Map<Integer, String>> topologyTaskToComponent = new HashMap<String, Map<Integer, String>>();
        Map<String, MetricInfo> metricInfoMap = new HashMap<String, MetricInfo>();
        for (Entry<String, Assignment> entry : assignments.entrySet()) {
            String topologyId = entry.getKey();
            Assignment assignment = entry.getValue();
            Set<ResourceWorkerSlot> workers = assignment.getWorkers();
            for (ResourceWorkerSlot worker : workers) {
                if (!id.equals(worker.getNodeId())) {
                    continue;
                }
                usedSlotNumber++;
                Integer port = worker.getPort();
                WorkerSummary workerSummary = portWorkerSummarys.get(port);
                if (workerSummary == null) {
                    workerSummary = new WorkerSummary();
                    workerSummary.set_port(port);
                    workerSummary.set_topology(topologyId);
                    workerSummary.set_tasks(new ArrayList<TaskComponent>());
                    portWorkerSummarys.put(port, workerSummary);
                }
                Map<Integer, String> taskToComponent = topologyTaskToComponent.get(topologyId);
                if (taskToComponent == null) {
                    taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
                    topologyTaskToComponent.put(topologyId, taskToComponent);
                }
                int earliest = TimeUtils.current_time_secs();
                for (Integer taskId : worker.getTasks()) {
                    TaskComponent taskComponent = new TaskComponent();
                    taskComponent.set_component(taskToComponent.get(taskId));
                    taskComponent.set_taskId(taskId);
                    Integer startTime = assignment.getTaskStartTimeSecs().get(taskId);
                    if (startTime != null && startTime < earliest) {
                        earliest = startTime;
                    }
                    workerSummary.add_to_tasks(taskComponent);
                }
                workerSummary.set_uptime(TimeUtils.time_delta(earliest));
                String workerSlotName = getWorkerSlotName(ip, port);
                List<MetricInfo> workerMetricInfoList = this.data.getMetricCache().getMetricData(topologyId, MetaType.WORKER);
                if (workerMetricInfoList.size() > 0) {
                    MetricInfo workerMetricInfo = workerMetricInfoList.get(0);
                    // remove metrics that don't belong to current worker
                    for (Iterator<String> itr = workerMetricInfo.get_metrics().keySet().iterator(); itr.hasNext(); ) {
                        String metricName = itr.next();
                        if (!metricName.contains(ip)) {
                            itr.remove();
                        }
                    }
                    metricInfoMap.put(workerSlotName, workerMetricInfo);
                }
            }
        }
        List<WorkerSummary> workerList = new ArrayList<WorkerSummary>();
        workerList.addAll(portWorkerSummarys.values());
        Map<String, Integer> supervisorToUsedSlotNum = new HashMap<String, Integer>();
        supervisorToUsedSlotNum.put(id, usedSlotNumber);
        SupervisorSummary supervisorSummary = NimbusUtils.mkSupervisorSummary(supervisorInfo, id, supervisorToUsedSlotNum);
        return new SupervisorWorkers(supervisorSummary, workerList, metricInfoMap);
    } catch (TException e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw e;
    } catch (Exception e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw new TException(e);
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getSupervisorWorkers", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 4 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class TopologyAssign method getAliveSupervsByHb.

private void getAliveSupervsByHb(Map<String, SupervisorInfo> supervisorInfos, Map conf) {
    int currentTime = TimeUtils.current_time_secs();
    int hbTimeout = JStormUtils.parseInt(conf.get(Config.NIMBUS_SUPERVISOR_TIMEOUT_SECS), (JStormUtils.MIN_1 * 3));
    Set<String> supervisorTobeRemoved = new HashSet<String>();
    for (Entry<String, SupervisorInfo> entry : supervisorInfos.entrySet()) {
        SupervisorInfo supInfo = entry.getValue();
        int lastReportTime = supInfo.getTimeSecs();
        if ((currentTime - lastReportTime) > hbTimeout) {
            LOG.warn("Supervisor-" + supInfo.getHostName() + " is dead. lastReportTime=" + lastReportTime);
            supervisorTobeRemoved.add(entry.getKey());
        }
    }
    for (String name : supervisorTobeRemoved) {
        supervisorInfos.remove(name);
    }
}

Example 5 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class NimbusUtils method mkSupervisorSummaries.

public static List<SupervisorSummary> mkSupervisorSummaries(Map<String, SupervisorInfo> supervisorInfos, Map<String, Assignment> assignments) {
    Map<String, Integer> supervisorToLeftSlotNum = new HashMap<String, Integer>();
    for (Entry<String, Assignment> entry : assignments.entrySet()) {
        Set<ResourceWorkerSlot> workers = entry.getValue().getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            String supervisorId = worker.getNodeId();
            SupervisorInfo supervisorInfo = supervisorInfos.get(supervisorId);
            if (supervisorInfo == null) {
                continue;
            }
            Integer slots = supervisorToLeftSlotNum.get(supervisorId);
            if (slots == null) {
                slots = 0;
                supervisorToLeftSlotNum.put(supervisorId, slots);
            }
            supervisorToLeftSlotNum.put(supervisorId, ++slots);
        }
    }
    List<SupervisorSummary> ret = new ArrayList<SupervisorSummary>();
    for (Entry<String, SupervisorInfo> entry : supervisorInfos.entrySet()) {
        String supervisorId = entry.getKey();
        SupervisorInfo supervisorInfo = entry.getValue();
        SupervisorSummary summary = mkSupervisorSummary(supervisorInfo, supervisorId, supervisorToLeftSlotNum);
        ret.add(summary);
    }
    Collections.sort(ret, new Comparator<SupervisorSummary>() {

        @Override
        public int compare(SupervisorSummary o1, SupervisorSummary o2) {
            return o1.get_host().compareTo(o2.get_host());
        }
    });
    return ret;
}