Examples with ClusterSummary backtype.storm.generated.ClusterSummary used on opensource projects

Example 1 with ClusterSummary

use of backtype.storm.generated.ClusterSummary in project jstorm by alibaba.

the class JStormHelper method getSupervisorHosts.

public static List<String> getSupervisorHosts() throws Exception {
    try {
        List<String> hosts = new ArrayList<>();
        NimbusClient client = getNimbusClient(null);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
        Collections.sort(supervisorSummaries, new Comparator<SupervisorSummary>() {

            @Override
            public int compare(SupervisorSummary o1, SupervisorSummary o2) {
                // TODO Auto-generated method stub
                int o1Left = o1.get_numWorkers() - o1.get_numUsedWorkers();
                int o2Left = o2.get_numWorkers() - o2.get_numUsedWorkers();
                return o1Left - o2Left;
            }
        });
        for (SupervisorSummary supervisorSummary : supervisorSummaries) {
            hosts.add(supervisorSummary.get_host());
        }
        return hosts;
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;
        }
        LOG.error("Failed to kill all topology ", e);
        throw new RuntimeException(e);
    }
}

Example 2 with ClusterSummary

use of backtype.storm.generated.ClusterSummary in project jstorm by alibaba.

the class Monitor method getComponents.

/**
     * @@@ Don't be compatible with Storm
     * 
     *     Here skip the logic
     * @param client
     * @param topology
     * @return
     * @throws Exception
     */
private HashSet<String> getComponents(Nimbus.Client client, String topology) throws Exception {
    HashSet<String> components = new HashSet<String>();
    ClusterSummary clusterSummary = client.getClusterInfo();
    TopologySummary topologySummary = null;
    for (TopologySummary ts : clusterSummary.get_topologies()) {
        if (topology.equals(ts.get_name())) {
            topologySummary = ts;
            break;
        }
    }
    if (topologySummary == null) {
        throw new IllegalArgumentException("topology: " + topology + " not found");
    } else {
        String id = topologySummary.get_id();
    // GetInfoOptions getInfoOpts = new GetInfoOptions();
    // getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
    // TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
    // for (ExecutorSummary es: info.get_executors()) {
    // components.add(es.get_component_id());
    // }
    }
    return components;
}

Example 3 with ClusterSummary

use of backtype.storm.generated.ClusterSummary in project jstorm by alibaba.

the class Monitor method metrics.

public void metrics(Nimbus.Client client, long now, MetricsState state) throws Exception {
    long totalStatted = 0;
    int componentParallelism = 0;
    boolean streamFound = false;
    ClusterSummary clusterSummary = client.getClusterInfo();
    TopologySummary topologySummary = null;
    for (TopologySummary ts : clusterSummary.get_topologies()) {
        if (_topology.equals(ts.get_name())) {
            topologySummary = ts;
            break;
        }
    }
    if (topologySummary == null) {
        throw new IllegalArgumentException("topology: " + _topology + " not found");
    } else {
    // String id = topologySummary.get_id();
    // GetInfoOptions getInfoOpts = new GetInfoOptions();
    // getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
    // TopologyInfo info = client.getTopologyInfoWithOpts(id, getInfoOpts);
    // for (ExecutorSummary es: info.get_executors()) {
    // if (_component.equals(es.get_component_id())) {
    // componentParallelism ++;
    // ExecutorStats stats = es.get_stats();
    // if (stats != null) {
    // Map<String,Map<String,Long>> statted =
    // WATCH_EMITTED.equals(_watch) ? stats.get_emitted() : stats.get_transferred();
    // if ( statted != null) {
    // Map<String, Long> e2 = statted.get(":all-time");
    // if (e2 != null) {
    // Long stream = e2.get(_stream);
    // if (stream != null){
    // streamFound = true;
    // totalStatted += stream;
    // }
    // }
    // }
    // }
    // }
    // }
    }
    if (componentParallelism <= 0) {
        HashSet<String> components = getComponents(client, _topology);
        System.out.println("Available components for " + _topology + " :");
        System.out.println("------------------");
        for (String comp : components) {
            System.out.println(comp);
        }
        System.out.println("------------------");
        throw new IllegalArgumentException("component: " + _component + " not found");
    }
    if (!streamFound) {
        throw new IllegalArgumentException("stream: " + _stream + " not found");
    }
    long timeDelta = now - state.getLastTime();
    long stattedDelta = totalStatted - state.getLastStatted();
    state.setLastTime(now);
    state.setLastStatted(totalStatted);
    double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double) stattedDelta / (double) timeDelta);
    System.out.println(_topology + "\t" + _component + "\t" + componentParallelism + "\t" + _stream + "\t" + timeDelta + "\t" + stattedDelta + "\t" + throughput);
}

Example 4 with ClusterSummary

use of backtype.storm.generated.ClusterSummary in project jstorm by alibaba.

the class ClusterInfoBolt method getClusterInfo.

private void getClusterInfo(Client client) {
    try {
        ClusterSummary clusterSummary = client.getClusterInfo();
        List<SupervisorSummary> supervisorSummaryList = clusterSummary.get_supervisors();
        int totalWorkers = 0;
        int usedWorkers = 0;
        for (SupervisorSummary summary : supervisorSummaryList) {
            totalWorkers += summary.get_num_workers();
            usedWorkers += summary.get_num_used_workers();
        }
        int freeWorkers = totalWorkers - usedWorkers;
        LOGGER.info("cluster totalWorkers = " + totalWorkers + ", usedWorkers = " + usedWorkers + ", freeWorkers  = " + freeWorkers);
        HttpCatClient.sendMetric("ClusterMonitor", "freeSlots", "avg", String.valueOf(freeWorkers));
        HttpCatClient.sendMetric("ClusterMonitor", "totalSlots", "avg", String.valueOf(totalWorkers));
        List<TopologySummary> topologySummaryList = clusterSummary.get_topologies();
        long clusterTPS = 0l;
        for (TopologySummary topology : topologySummaryList) {
            long topologyTPS = getTopologyTPS(topology, client);
            clusterTPS += topologyTPS;
            if (topology.get_name().startsWith("ClusterMonitor")) {
                continue;
            }
            HttpCatClient.sendMetric(topology.get_name(), topology.get_name() + "-TPS", "avg", String.valueOf(topologyTPS));
        }
        HttpCatClient.sendMetric("ClusterMonitor", "ClusterEmitTPS", "avg", String.valueOf(clusterTPS));
    } catch (TException e) {
        initClient(configMap);
        LOGGER.error("get client info error.", e);
    } catch (NotAliveException nae) {
        LOGGER.warn("topology is dead.", nae);
    }
}

Example 5 with ClusterSummary

use of backtype.storm.generated.ClusterSummary in project jstorm by alibaba.

the class list method main.

public static void main(String[] args) {
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);
        if (args.length > 0 && !StringUtils.isBlank(args[0])) {
            String topologyName = args[0];
            TopologyInfo info = client.getClient().getTopologyInfoByName(topologyName);
            System.out.println("Successfully get topology info \n" + Utils.toPrettyJsonString(info));
        } else {
            ClusterSummary clusterSummary = client.getClient().getClusterInfo();
            System.out.println("Successfully get cluster info \n" + Utils.toPrettyJsonString(clusterSummary));
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}