Examples with SupervisorSummary backtype.storm.generated.SupervisorSummary used on opensource projects

Example 1 with SupervisorSummary

use of backtype.storm.generated.SupervisorSummary in project jstorm by alibaba.

the class JStormHelper method getSupervisorHosts.

public static List<String> getSupervisorHosts() throws Exception {
    try {
        List<String> hosts = new ArrayList<>();
        NimbusClient client = getNimbusClient(null);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
        Collections.sort(supervisorSummaries, new Comparator<SupervisorSummary>() {

            @Override
            public int compare(SupervisorSummary o1, SupervisorSummary o2) {
                int o1Left = o1.get_numWorkers() - o1.get_numUsedWorkers();
                int o2Left = o2.get_numWorkers() - o2.get_numUsedWorkers();
                return o1Left - o2Left;
            }
        });
        for (SupervisorSummary supervisorSummary : supervisorSummaries) {
            hosts.add(supervisorSummary.get_host());
        }
        return hosts;
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;
        }
        LOG.error("Failed to kill all topologies ", e);
        throw new RuntimeException(e);
    }
}

Example 2 with SupervisorSummary

use of backtype.storm.generated.SupervisorSummary in project jstorm by alibaba.

the class ServiceHandler method getSupervisorWorkersByHostOrId.

/**
 * gets supervisor workers by host or supervisor id, note that id priors to host.
 *
 * @param host host
 * @param id   supervisor id
 * @return supervisor workers
 */
private SupervisorWorkers getSupervisorWorkersByHostOrId(String host, String id) throws TException {
    long start = System.nanoTime();
    if (StringUtils.isBlank(id) && StringUtils.isBlank(host)) {
        throw new TException("Must specify host or supervisor id!");
    }
    try {
        StormClusterState stormClusterState = data.getStormClusterState();
        // all supervisors
        Map<String, SupervisorInfo> supervisorInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
        SupervisorInfo supervisorInfo = null;
        String ip;
        if (!StringUtils.isBlank(id)) {
            supervisorInfo = supervisorInfos.get(id);
            host = supervisorInfo.getHostName();
            ip = NetWorkUtils.host2Ip(host);
        } else {
            ip = NetWorkUtils.host2Ip(host);
            for (Entry<String, SupervisorInfo> entry : supervisorInfos.entrySet()) {
                SupervisorInfo info = entry.getValue();
                if (info.getHostName().equals(host) || info.getHostName().equals(ip)) {
                    id = entry.getKey();
                    supervisorInfo = info;
                    break;
                }
            }
        }
        if (supervisorInfo == null) {
            throw new TException("unknown supervisor id:" + id);
        }
        Map<String, Assignment> assignments = Cluster.get_all_assignment(stormClusterState, null);
        Map<Integer, WorkerSummary> portWorkerSummaries = new TreeMap<>();
        int usedSlotNumber = 0;
        Map<String, Map<Integer, String>> topologyTaskToComponent = new HashMap<>();
        Map<String, MetricInfo> metricInfoMap = new HashMap<>();
        for (Entry<String, Assignment> entry : assignments.entrySet()) {
            String topologyId = entry.getKey();
            Assignment assignment = entry.getValue();
            Set<ResourceWorkerSlot> workers = assignment.getWorkers();
            for (ResourceWorkerSlot worker : workers) {
                if (!id.equals(worker.getNodeId())) {
                    continue;
                }
                usedSlotNumber++;
                Integer port = worker.getPort();
                WorkerSummary workerSummary = portWorkerSummaries.get(port);
                if (workerSummary == null) {
                    workerSummary = new WorkerSummary();
                    workerSummary.set_port(port);
                    workerSummary.set_topology(topologyId);
                    workerSummary.set_tasks(new ArrayList<TaskComponent>());
                    portWorkerSummaries.put(port, workerSummary);
                }
                Map<Integer, String> taskToComponent = topologyTaskToComponent.get(topologyId);
                if (taskToComponent == null) {
                    taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
                    topologyTaskToComponent.put(topologyId, taskToComponent);
                }
                int earliest = TimeUtils.current_time_secs();
                for (Integer taskId : worker.getTasks()) {
                    TaskComponent taskComponent = new TaskComponent();
                    taskComponent.set_component(taskToComponent.get(taskId));
                    taskComponent.set_taskId(taskId);
                    Integer startTime = assignment.getTaskStartTimeSecs().get(taskId);
                    if (startTime != null && startTime < earliest) {
                        earliest = startTime;
                    }
                    workerSummary.add_to_tasks(taskComponent);
                }
                workerSummary.set_uptime(TimeUtils.time_delta(earliest));
                String workerSlotName = getWorkerSlotName(ip, port);
                List<MetricInfo> workerMetricInfoList = this.data.getMetricCache().getMetricData(topologyId, MetaType.WORKER);
                if (workerMetricInfoList.size() > 0) {
                    MetricInfo workerMetricInfo = workerMetricInfoList.get(0);
                    // remove metrics that don't belong to current worker
                    for (Iterator<String> itr = workerMetricInfo.get_metrics().keySet().iterator(); itr.hasNext(); ) {
                        String metricName = itr.next();
                        if (!metricName.contains(ip)) {
                            itr.remove();
                        }
                    }
                    metricInfoMap.put(workerSlotName, workerMetricInfo);
                }
            }
        }
        List<WorkerSummary> workerList = new ArrayList<>();
        workerList.addAll(portWorkerSummaries.values());
        Map<String, Integer> supervisorToUsedSlotNum = new HashMap<>();
        supervisorToUsedSlotNum.put(id, usedSlotNumber);
        SupervisorSummary supervisorSummary = NimbusUtils.mkSupervisorSummary(supervisorInfo, id, supervisorToUsedSlotNum);
        return new SupervisorWorkers(supervisorSummary, workerList, metricInfoMap);
    } catch (TException e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw e;
    } catch (Exception e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw new TException(e);
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getSupervisorWorkers", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 3 with SupervisorSummary

use of backtype.storm.generated.SupervisorSummary in project jstorm by alibaba.

the class NimbusUtils method mkSupervisorSummaries.

public static List<SupervisorSummary> mkSupervisorSummaries(Map<String, SupervisorInfo> supervisorInfos, Map<String, Assignment> assignments) {
    Map<String, Integer> supervisorToLeftSlotNum = new HashMap<>();
    for (Entry<String, Assignment> entry : assignments.entrySet()) {
        Set<ResourceWorkerSlot> workers = entry.getValue().getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            String supervisorId = worker.getNodeId();
            SupervisorInfo supervisorInfo = supervisorInfos.get(supervisorId);
            if (supervisorInfo == null) {
                continue;
            }
            Integer slots = supervisorToLeftSlotNum.get(supervisorId);
            if (slots == null) {
                slots = 0;
                supervisorToLeftSlotNum.put(supervisorId, slots);
            }
            supervisorToLeftSlotNum.put(supervisorId, ++slots);
        }
    }
    List<SupervisorSummary> ret = new ArrayList<>();
    for (Entry<String, SupervisorInfo> entry : supervisorInfos.entrySet()) {
        String supervisorId = entry.getKey();
        SupervisorInfo supervisorInfo = entry.getValue();
        SupervisorSummary summary = mkSupervisorSummary(supervisorInfo, supervisorId, supervisorToLeftSlotNum);
        ret.add(summary);
    }
    Collections.sort(ret, new Comparator<SupervisorSummary>() {

        @Override
        public int compare(SupervisorSummary o1, SupervisorSummary o2) {
            return o1.get_host().compareTo(o2.get_host());
        }
    });
    return ret;
}

Example 4 with SupervisorSummary

use of backtype.storm.generated.SupervisorSummary in project jstorm by alibaba.

the class ClusterInfoBolt method getClusterInfo.

private void getClusterInfo(Client client) {
    try {
        ClusterSummary clusterSummary = client.getClusterInfo();
        List<SupervisorSummary> supervisorSummaryList = clusterSummary.get_supervisors();
        int totalWorkers = 0;
        int usedWorkers = 0;
        for (SupervisorSummary summary : supervisorSummaryList) {
            totalWorkers += summary.get_num_workers();
            usedWorkers += summary.get_num_used_workers();
        }
        int freeWorkers = totalWorkers - usedWorkers;
        LOGGER.info("cluster totalWorkers = " + totalWorkers + ", usedWorkers = " + usedWorkers + ", freeWorkers  = " + freeWorkers);
        HttpCatClient.sendMetric("ClusterMonitor", "freeSlots", "avg", String.valueOf(freeWorkers));
        HttpCatClient.sendMetric("ClusterMonitor", "totalSlots", "avg", String.valueOf(totalWorkers));
        List<TopologySummary> topologySummaryList = clusterSummary.get_topologies();
        long clusterTPS = 0l;
        for (TopologySummary topology : topologySummaryList) {
            long topologyTPS = getTopologyTPS(topology, client);
            clusterTPS += topologyTPS;
            if (topology.get_name().startsWith("ClusterMonitor")) {
                continue;
            }
            HttpCatClient.sendMetric(topology.get_name(), topology.get_name() + "-TPS", "avg", String.valueOf(topologyTPS));
        }
        HttpCatClient.sendMetric("ClusterMonitor", "ClusterEmitTPS", "avg", String.valueOf(clusterTPS));
    } catch (TException e) {
        initClient(configMap);
        LOGGER.error("get client info error.", e);
    } catch (NotAliveException nae) {
        LOGGER.warn("topology is dead.", nae);
    }
}

Example 5 with SupervisorSummary

use of backtype.storm.generated.SupervisorSummary in project jstorm by alibaba.

the class NimbusUtils method getNimbusSummary.

public static NimbusSummary getNimbusSummary(StormClusterState stormClusterState, List<SupervisorSummary> supervisorSummaries, NimbusData data) throws Exception {
    NimbusSummary ret = new NimbusSummary();
    String master = stormClusterState.get_leader_host();
    NimbusStat nimbusMaster = new NimbusStat();
    nimbusMaster.set_host(master);
    nimbusMaster.set_uptimeSecs(String.valueOf(data.uptime()));
    ret.set_nimbusMaster(nimbusMaster);
    List<NimbusStat> nimbusSlaveList = new ArrayList<>();
    ret.set_nimbusSlaves(nimbusSlaveList);
    Map<String, String> nimbusSlaveMap = Cluster.get_all_nimbus_slave(stormClusterState);
    if (nimbusSlaveMap != null) {
        for (Entry<String, String> entry : nimbusSlaveMap.entrySet()) {
            NimbusStat slave = new NimbusStat();
            slave.set_host(entry.getKey());
            slave.set_uptimeSecs(entry.getValue());
            nimbusSlaveList.add(slave);
        }
    }
    int totalPort = 0;
    int usedPort = 0;
    for (SupervisorSummary supervisor : supervisorSummaries) {
        totalPort += supervisor.get_numWorkers();
        usedPort += supervisor.get_numUsedWorkers();
    }
    ret.set_supervisorNum(supervisorSummaries.size());
    ret.set_totalPortNum(totalPort);
    ret.set_usedPortNum(usedPort);
    ret.set_freePortNum(totalPort - usedPort);
    ret.set_version(Utils.getVersion());
    return ret;
}