Examples with MetricSnapshot backtype.storm.generated.MetricSnapshot used on opensource projects

Example 16 with MetricSnapshot

use of backtype.storm.generated.MetricSnapshot in project jstorm by alibaba.

the class UIMetricUtils method getComponentMetrics.

/**
     * get all component metrics in the topology
     * @param info metric info
     * @param window window duration for metrics in seconds
     * @param componentSummaries list of component summaries
     * @param userDefinedMetrics list of user defined metrics for return
     * @return list of component metrics
     */
public static List<UIComponentMetric> getComponentMetrics(MetricInfo info, int window, List<ComponentSummary> componentSummaries, List<UIUserDefinedMetric> userDefinedMetrics) {
    Map<String, UIComponentMetric> componentData = new HashMap<>();
    if (info != null) {
        for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
            String name = metric.getKey();
            String[] split_name = name.split("@");
            String compName = UIMetricUtils.extractComponentName(split_name);
            String metricName = UIMetricUtils.extractMetricName(split_name);
            String group = UIMetricUtils.extractGroup(split_name);
            String parentComp = null;
            if (metricName != null && metricName.contains(".")) {
                parentComp = metricName.split("\\.")[0];
                metricName = metricName.split("\\.")[1];
            }
            if (!metric.getValue().containsKey(window)) {
                LOG.debug("component snapshot {} missing window:{}", metric.getKey(), window);
                continue;
            }
            MetricSnapshot snapshot = metric.getValue().get(window);
            if (group != null && group.equals("udf")) {
                UIUserDefinedMetric udm = new UIUserDefinedMetric(metricName, compName);
                udm.setValue(UIMetricUtils.getMetricValue(snapshot));
                udm.setType(snapshot.get_metricType());
                userDefinedMetrics.add(udm);
            } else {
                UIComponentMetric compMetric;
                if (componentData.containsKey(compName)) {
                    compMetric = componentData.get(compName);
                } else {
                    compMetric = new UIComponentMetric(compName);
                    componentData.put(compName, compMetric);
                }
                compMetric.setMetricValue(snapshot, parentComp, metricName);
            }
        }
    }
    //merge sub metrics
    for (UIComponentMetric comp : componentData.values()) {
        comp.mergeValue();
    }
    //combine the summary info into metrics
    TreeMap<String, UIComponentMetric> ret = new TreeMap<>();
    for (ComponentSummary summary : componentSummaries) {
        String compName = summary.get_name();
        UIComponentMetric compMetric;
        if (componentData.containsKey(compName)) {
            compMetric = componentData.get(compName);
            compMetric.setParallel(summary.get_parallel());
            compMetric.setType(summary.get_type());
            compMetric.setErrors(summary.get_errors());
        } else {
            compMetric = new UIComponentMetric(compName, summary.get_parallel(), summary.get_type());
            compMetric.setErrors(summary.get_errors());
            componentData.put(compName, compMetric);
        }
        String key = compMetric.getType() + compName;
        if (compName.startsWith("__")) {
            key = "a" + key;
        }
        compMetric.setSortedKey(key);
        ret.put(key, compMetric);
    }
    return new ArrayList<>(ret.descendingMap().values());
}

Example 17 with MetricSnapshot

use of backtype.storm.generated.MetricSnapshot in project jstorm by alibaba.

the class MergeEvent method mergeAndUploadClusterMetrics.

private void mergeAndUploadClusterMetrics() {
    TopologyMetricContext clusterContext = context.getClusterTopologyMetricContext();
    TopologyMetric tpMetric = clusterContext.mergeMetrics();
    if (tpMetric == null) {
        tpMetric = MetricUtils.mkTopologyMetric();
        tpMetric.set_topologyMetric(MetricUtils.mkMetricInfo());
    }
    //reset snapshots metric id
    MetricInfo clusterMetrics = tpMetric.get_topologyMetric();
    Map<String, Long> metricName2Id = clusterContext.getMemMeta();
    for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : clusterMetrics.get_metrics().entrySet()) {
        String metricName = entry.getKey();
        MetricType metricType = MetricUtils.metricType(metricName);
        Long metricId = metricName2Id.get(metricName);
        for (Map.Entry<Integer, MetricSnapshot> metric : entry.getValue().entrySet()) {
            MetricSnapshot snapshot = metric.getValue();
            snapshot.set_metricId(metricId);
            if (metricType == MetricType.HISTOGRAM) {
                snapshot.set_points(new byte[0]);
            }
        //                entry.getValue().put(metric.getKey(), snapshot);
        }
    }
    //fill the unacquired metrics with zero
    long ts = System.currentTimeMillis();
    for (Map.Entry<String, Long> entry : metricName2Id.entrySet()) {
        String name = entry.getKey();
        if (!clusterMetrics.get_metrics().containsKey(name)) {
            Map<Integer, MetricSnapshot> metric = new HashMap<>();
            MetricType type = MetricUtils.metricType(name);
            metric.put(AsmWindow.M1_WINDOW, new MetricSnapshot(entry.getValue(), ts, type.getT()));
            clusterMetrics.put_to_metrics(name, metric);
        }
    }
    //upload to cache
    UpdateEvent.pushEvent(JStormMetrics.CLUSTER_METRIC_KEY, tpMetric);
    LOG.debug("send update event for cluster metrics, size : {}", clusterMetrics.get_metrics_size());
}

Example 18 with MetricSnapshot

use of backtype.storm.generated.MetricSnapshot in project jstorm by alibaba.

the class MetricUtils method convert.

public static MetricSnapshot convert(AsmMeterSnapshot snapshot) {
    MetricSnapshot ret = new MetricSnapshot();
    ret.set_metricId(snapshot.getMetricId());
    ret.set_ts(TimeUtils.alignTimeToMin(snapshot.getTs()));
    ret.set_metricType(MetricType.METER.getT());
    ret.set_m1(snapshot.getM1());
    ret.set_m5(snapshot.getM5());
    ret.set_m15(snapshot.getM15());
    ret.set_mean(snapshot.getMean());
    return ret;
}

Example 19 with MetricSnapshot

use of backtype.storm.generated.MetricSnapshot in project jstorm by alibaba.

the class MetricUtils method str.

public static String str(Object obj) {
    if (obj instanceof MetricSnapshot) {
        MetricSnapshot snapshot = (MetricSnapshot) obj;
        MetricType type = MetricType.parse(snapshot.get_metricType());
        if (type == MetricType.COUNTER) {
            return counterStr(snapshot);
        } else if (type == MetricType.GAUGE) {
            return gaugeStr(snapshot);
        } else if (type == MetricType.METER) {
            return meterStr(snapshot);
        } else if (type == MetricType.HISTOGRAM) {
            return histogramStr(snapshot);
        }
    }
    return obj.toString();
}

Example 20 with MetricSnapshot

use of backtype.storm.generated.MetricSnapshot in project jstorm by alibaba.

the class TopologyMetricContext method mergeMetrics.

public TopologyMetric mergeMetrics() {
    long start = System.currentTimeMillis();
    if (getMemCache().size() == 0) {
        //LOG.info("topology:{}, metric size is 0, skip...", topologyId);
        return null;
    }
    if (isMerging()) {
        LOG.info("topology {} is already merging, skip...", topologyId);
        return null;
    }
    setMerging(true);
    try {
        Map<String, MetricInfo> workerMetricMap = this.memCache;
        // reset mem cache
        this.memCache = new ConcurrentHashMap<>();
        MetricInfo topologyMetrics = MetricUtils.mkMetricInfo();
        MetricInfo componentMetrics = MetricUtils.mkMetricInfo();
        MetricInfo taskMetrics = MetricUtils.mkMetricInfo();
        MetricInfo streamMetrics = MetricUtils.mkMetricInfo();
        MetricInfo workerMetrics = MetricUtils.mkMetricInfo();
        MetricInfo nettyMetrics = MetricUtils.mkMetricInfo();
        TopologyMetric tpMetric = new TopologyMetric(topologyMetrics, componentMetrics, workerMetrics, taskMetrics, streamMetrics, nettyMetrics);
        // metric name => worker count
        Map<String, Integer> metricNameCounters = new HashMap<>();
        // special for histograms & timers, we merge the points to get a new snapshot data.
        Map<String, Map<Integer, Histogram>> histograms = new HashMap<>();
        // iterate metrics of all workers within the same topology
        for (ConcurrentMap.Entry<String, MetricInfo> metricEntry : workerMetricMap.entrySet()) {
            MetricInfo metricInfo = metricEntry.getValue();
            // merge counters: add old and new values, note we only add incoming new metrics and overwrite
            // existing data, same for all below.
            Map<String, Map<Integer, MetricSnapshot>> metrics = metricInfo.get_metrics();
            for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : metrics.entrySet()) {
                String metricName = metric.getKey();
                Map<Integer, MetricSnapshot> data = metric.getValue();
                MetaType metaType = MetricUtils.metaType(metricName);
                MetricType metricType = MetricUtils.metricType(metricName);
                if (metricType == MetricType.COUNTER) {
                    mergeCounters(tpMetric, metaType, metricName, data);
                } else if (metricType == MetricType.GAUGE) {
                    mergeGauges(tpMetric, metaType, metricName, data);
                } else if (metricType == MetricType.METER) {
                    mergeMeters(getMetricInfoByType(tpMetric, metaType), metricName, data, metricNameCounters);
                } else if (metricType == MetricType.HISTOGRAM) {
                    mergeHistograms(getMetricInfoByType(tpMetric, metaType), metricName, data, metricNameCounters, histograms);
                }
            }
        }
        adjustHistogramTimerMetrics(tpMetric, metricNameCounters, histograms);
        // for counters, we only report delta data every time, need to sum with old data
        //adjustCounterMetrics(tpMetric, oldTpMetric);
        LOG.info("merge topology metrics:{}, cost:{}", topologyId, System.currentTimeMillis() - start);
        // debug logs
        MetricUtils.printMetricInfo(tpMetric.get_topologyMetric());
        return tpMetric;
    } finally {
        setMerging(false);
    }
}