Examples with MetricInfo backtype.storm.generated.MetricInfo used on opensource projects

Example 11 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class ClusterMetricsContext method getTopologyMetric.

/**
     * get topology metrics, note that only topology & component & worker
     * metrics are returned
     */
public TopologyMetric getTopologyMetric(String topologyId) {
    long start = System.nanoTime();
    try {
        TopologyMetric ret = new TopologyMetric();
        List<MetricInfo> topologyMetrics = metricCache.getMetricData(topologyId, MetaType.TOPOLOGY);
        List<MetricInfo> componentMetrics = metricCache.getMetricData(topologyId, MetaType.COMPONENT);
        List<MetricInfo> workerMetrics = metricCache.getMetricData(topologyId, MetaType.WORKER);
        MetricInfo dummy = MetricUtils.mkMetricInfo();
        if (topologyMetrics.size() > 0) {
            // get the last min topology metric
            ret.set_topologyMetric(topologyMetrics.get(topologyMetrics.size() - 1));
        } else {
            ret.set_topologyMetric(dummy);
        }
        if (componentMetrics.size() > 0) {
            ret.set_componentMetric(componentMetrics.get(0));
        } else {
            ret.set_componentMetric(dummy);
        }
        if (workerMetrics.size() > 0) {
            ret.set_workerMetric(workerMetrics.get(0));
        } else {
            ret.set_workerMetric(dummy);
        }
        ret.set_taskMetric(dummy);
        ret.set_streamMetric(dummy);
        ret.set_nettyMetric(dummy);
        return ret;
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getTopologyMetric", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 12 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class MetricsUploader method batchUploadMetrics.

private void batchUploadMetrics(MetricInfo metricInfo, MetaType metaType) throws Exception {
    if (metricInfo.get_metrics_size() > MAX_BATCH_SIZE) {
        Map<String, Map<Integer, MetricSnapshot>> data = metricInfo.get_metrics();
        Map<String, Map<Integer, MetricSnapshot>> part = Maps.newHashMapWithExpectedSize(MAX_BATCH_SIZE);
        MetricInfo uploadPart = new MetricInfo();
        int i = 0;
        for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : data.entrySet()) {
            part.put(entry.getKey(), entry.getValue());
            if (++i >= MAX_BATCH_SIZE) {
                uploadPart.set_metrics(part);
                uploadParts(uploadPart, metaType);
                i = 0;
                part.clear();
            }
        }
        if (part.size() > 0) {
            uploadPart.set_metrics(part);
            uploadParts(uploadPart, metaType);
        }
    } else {
        uploadParts(metricInfo, metaType);
    }
}

Example 13 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class NettyController method show.

@RequestMapping(value = "/netty", method = RequestMethod.GET)
public String show(@RequestParam(value = "cluster", required = true) String clusterName, @RequestParam(value = "topology", required = true) String topology_id, @RequestParam(value = "host", required = true) String host, @RequestParam(value = "page", required = false) String page, @RequestParam(value = "win", required = false) String win, ModelMap model) {
    clusterName = StringEscapeUtils.escapeHtml(clusterName);
    long start = System.currentTimeMillis();
    host = NetWorkUtils.host2Ip(host);
    int window = UIUtils.parseWindow(win);
    UIUtils.addWindowAttribute(model, window);
    int _page = JStormUtils.parseInt(page, 1);
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        //get netty metrics, page is from 1
        MetricInfo nettyMetric = client.getClient().getPagingNettyMetrics(topology_id, host, _page);
        //            System.out.println("nettyMetric:"+nettyMetric);
        List<UINettyMetric> nettyData = getNettyData(nettyMetric, host, window);
        model.addAttribute("nettyMetrics", nettyData);
        model.addAttribute("nettyHead", UIMetricUtils.sortHead(nettyData, UINettyMetric.HEAD));
        int size = client.getClient().getNettyMetricSizeByHost(topology_id, host);
        //            System.out.println("netty size:"+size);
        int pageSize = (size + PAGE_SIZE - 1) / PAGE_SIZE;
        model.addAttribute("pageSize", pageSize);
        model.addAttribute("curPage", _page);
        if (size == 0) {
            //check whether netty metric is turn off
            Map conf = UIUtils.getTopologyConf(clusterName, topology_id);
            boolean enable = MetricUtils.isEnableNettyMetrics(conf);
            if (!enable) {
                model.addAttribute("flush", "the netty metrics is not enabled");
            }
        }
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        UIUtils.addErrorAttribute(model, e);
    }
    // page information
    model.addAttribute("clusterName", clusterName);
    model.addAttribute("host", host);
    model.addAttribute("page", "netty");
    model.addAttribute("topologyId", topology_id);
    model.addAttribute("supervisorPort", UIUtils.getSupervisorPort(clusterName));
    UIUtils.addTitleAttribute(model, "Netty Summary");
    LOG.info("netty page show cost:{}ms", System.currentTimeMillis() - start);
    return "netty";
}

Example 14 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class SupervisorController method show.

@RequestMapping(value = "/supervisor", method = RequestMethod.GET)
public String show(@RequestParam(value = "cluster", required = true) String clusterName, @RequestParam(value = "host", required = true) String host, @RequestParam(value = "win", required = false) String win, ModelMap model) {
    clusterName = StringEscapeUtils.escapeHtml(clusterName);
    long start = System.currentTimeMillis();
    host = NetWorkUtils.host2Ip(host);
    int window = UIUtils.parseWindow(win);
    UIUtils.addWindowAttribute(model, window);
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        //get supervisor summary
        SupervisorWorkers supervisorWorkers = client.getClient().getSupervisorWorkers(host);
        model.addAttribute("supervisor", new SupervisorEntity(supervisorWorkers.get_supervisor()));
        //get worker summary
        List<WorkerSummary> workerSummaries = supervisorWorkers.get_workers();
        model.addAttribute("workerSummary", UIUtils.getWorkerEntities(workerSummaries));
        //get worker metrics
        Map<String, MetricInfo> workerMetricInfo = supervisorWorkers.get_workerMetric();
        List<UIWorkerMetric> workerMetrics = UIMetricUtils.getWorkerMetrics(workerMetricInfo, workerSummaries, host, window);
        //            System.out.println("workerMetricInfo:"+workerMetricInfo);
        model.addAttribute("workerMetrics", workerMetrics);
        model.addAttribute("workerHead", UIMetricUtils.sortHead(workerMetrics, UIWorkerMetric.HEAD));
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        UIUtils.addErrorAttribute(model, e);
    }
    // page information
    model.addAttribute("clusterName", clusterName);
    model.addAttribute("host", host);
    model.addAttribute("page", "supervisor");
    model.addAttribute("supervisorPort", UIUtils.getSupervisorPort(clusterName));
    UIUtils.addTitleAttribute(model, "Supervisor Summary");
    LOG.info("supervisor page show cost:{}ms", System.currentTimeMillis() - start);
    return "supervisor";
}

Example 15 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class UIMetricUtils method getWorkerMetrics.

public static List<UIWorkerMetric> getWorkerMetrics(Map<String, MetricInfo> workerMetricInfo, List<WorkerSummary> workerSummaries, String host, int window) {
    Map<String, UIWorkerMetric> workerMetrics = new HashMap<>();
    for (MetricInfo info : workerMetricInfo.values()) {
        if (info != null) {
            for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
                String name = metric.getKey();
                String[] split_name = name.split("@");
                String _host = UIMetricUtils.extractComponentName(split_name);
                if (!host.equals(_host))
                    continue;
                //only handle the specific host
                String port = UIMetricUtils.extractTaskId(split_name);
                String key = host + ":" + port;
                String metricName = UIMetricUtils.extractMetricName(split_name);
                MetricSnapshot snapshot = metric.getValue().get(window);
                UIWorkerMetric workerMetric;
                if (workerMetrics.containsKey(key)) {
                    workerMetric = workerMetrics.get(key);
                } else {
                    workerMetric = new UIWorkerMetric(host, port);
                    workerMetrics.put(key, workerMetric);
                }
                workerMetric.setMetricValue(snapshot, metricName);
            }
        }
    }
    for (WorkerSummary ws : workerSummaries) {
        String worker = host + ":" + ws.get_port();
        if (workerMetrics.containsKey(worker)) {
            workerMetrics.get(worker).setTopology(ws.get_topology());
        }
    }
    return new ArrayList<>(workerMetrics.values());
}