Examples with MetricInfo backtype.storm.generated.MetricInfo used on opensource projects

Example 26 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class TopologyMetricContext method mergeGauges.

public void mergeGauges(TopologyMetric tpMetric, MetaType metaType, String meta, Map<Integer, MetricSnapshot> data) {
    MetricInfo metricInfo = getMetricInfoByType(tpMetric, metaType);
    Map<Integer, MetricSnapshot> existing = metricInfo.get_metrics().get(meta);
    if (existing == null) {
        metricInfo.put_to_metrics(meta, data);
    } else {
        for (Map.Entry<Integer, MetricSnapshot> dataEntry : data.entrySet()) {
            Integer win = dataEntry.getKey();
            MetricSnapshot snapshot = dataEntry.getValue();
            MetricSnapshot old = existing.get(win);
            if (old == null) {
                existing.put(win, snapshot);
            } else {
                if (snapshot.get_ts() >= old.get_ts()) {
                    old.set_ts(snapshot.get_ts());
                    if (metaType != MetaType.TOPOLOGY) {
                        old.set_doubleValue(snapshot.get_doubleValue());
                    } else {
                        // for topology metric, gauge might be add-able, e.g., cpu, memory, etc.
                        old.set_doubleValue(old.get_doubleValue() + snapshot.get_doubleValue());
                    }
                }
            }
        }
    }
}

Example 27 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class MetricsUploader method uploadMetrics.

/**
     * upload metrics sequentially due to thrift frame size limit (15MB)
     */
private void uploadMetrics(TopologyMetric tpMetric) throws Exception {
    long start = System.currentTimeMillis();
    if (tpMetric == null) {
        return;
    }
    try {
        synchronized (lock) {
            if (client == null || !client.isValid()) {
                client = new NimbusClientWrapper();
                client.init(conf);
            }
        }
        MetricInfo topologyMetrics = tpMetric.get_topologyMetric();
        MetricInfo componentMetrics = tpMetric.get_componentMetric();
        MetricInfo taskMetrics = tpMetric.get_taskMetric();
        MetricInfo streamMetrics = tpMetric.get_streamMetric();
        MetricInfo workerMetrics = tpMetric.get_workerMetric();
        MetricInfo nettyMetrics = tpMetric.get_nettyMetric();
        int totalSize = topologyMetrics.get_metrics_size() + componentMetrics.get_metrics_size() + taskMetrics.get_metrics_size() + streamMetrics.get_metrics_size() + workerMetrics.get_metrics_size() + nettyMetrics.get_metrics_size();
        // pressure of nimbus
        if (totalSize < MAX_BATCH_SIZE) {
            client.getClient().uploadTopologyMetrics(topologyId, new TopologyMetric(topologyMetrics, componentMetrics, workerMetrics, taskMetrics, streamMetrics, nettyMetrics));
        } else {
            client.getClient().uploadTopologyMetrics(topologyId, new TopologyMetric(topologyMetrics, componentMetrics, dummy, dummy, dummy, dummy));
            batchUploadMetrics(workerMetrics, MetaType.WORKER);
            batchUploadMetrics(taskMetrics, MetaType.TASK);
            batchUploadMetrics(streamMetrics, MetaType.STREAM);
            batchUploadMetrics(nettyMetrics, MetaType.NETTY);
        }
    } catch (Exception e) {
        String errorInfo = "Failed to upload worker metrics";
        LOG.error("Failed to upload worker metrics ", e);
        if (client != null) {
            client.cleanup();
        }
        zkCluster.report_task_error(context.getTopologyId(), context.getThisTaskId(), errorInfo, ErrorConstants.WARN, ErrorConstants.CODE_USER);
    }
    metricLogger.info("upload metrics, cost:{}", System.currentTimeMillis() - start);
}

Example 28 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class UIMetricUtils method getStreamMetrics.

public static List<UIStreamMetric> getStreamMetrics(List<MetricInfo> taskStreamMetrics, String component, int id, int window) {
    Map<String, UIStreamMetric> streamData = new HashMap<>();
    if (taskStreamMetrics.size() > 1) {
        MetricInfo info = taskStreamMetrics.get(1);
        if (info != null) {
            for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
                String name = metric.getKey();
                String[] split_name = name.split("@");
                int taskId = JStormUtils.parseInt(UIMetricUtils.extractTaskId(split_name));
                if (taskId != id)
                    continue;
                //only handle the specific task
                String metricName = UIMetricUtils.extractMetricName(split_name);
                String streamId = UIMetricUtils.extractStreamId(split_name);
                String parentComp = null;
                if (metricName != null && metricName.contains(".")) {
                    parentComp = metricName.split("\\.")[0];
                    metricName = metricName.split("\\.")[1];
                }
                MetricSnapshot snapshot = metric.getValue().get(window);
                UIStreamMetric streamMetric;
                if (streamData.containsKey(streamId)) {
                    streamMetric = streamData.get(streamId);
                } else {
                    streamMetric = new UIStreamMetric(component, streamId);
                    streamData.put(streamId, streamMetric);
                }
                streamMetric.setMetricValue(snapshot, parentComp, metricName);
            }
        }
    }
    for (UIStreamMetric stream : streamData.values()) {
        stream.mergeValue();
    }
    return new ArrayList<>(streamData.values());
}

Example 29 with MetricInfo

use of backtype.storm.generated.MetricInfo in project jstorm by alibaba.

the class UIMetricUtils method getTaskMetric.

/**
     * get the specific task metric
     * @param taskStreamMetrics raw metric info
     * @param component component name
     * @param id task id
     * @param window window duration for metrics in seconds
     * @return the task metric
     */
public static UITaskMetric getTaskMetric(List<MetricInfo> taskStreamMetrics, String component, int id, int window) {
    UITaskMetric taskMetric = new UITaskMetric(component, id);
    if (taskStreamMetrics.size() > 1) {
        MetricInfo info = taskStreamMetrics.get(0);
        if (info != null) {
            for (Map.Entry<String, Map<Integer, MetricSnapshot>> metric : info.get_metrics().entrySet()) {
                String name = metric.getKey();
                String[] split_name = name.split("@");
                int taskId = JStormUtils.parseInt(UIMetricUtils.extractTaskId(split_name));
                if (taskId != id)
                    continue;
                //only handle the specific task
                String metricName = UIMetricUtils.extractMetricName(split_name);
                String parentComp = null;
                if (metricName != null && metricName.contains(".")) {
                    parentComp = metricName.split("\\.")[0];
                    metricName = metricName.split("\\.")[1];
                }
                MetricSnapshot snapshot = metric.getValue().get(window);
                taskMetric.setMetricValue(snapshot, parentComp, metricName);
            }
        }
    }
    taskMetric.mergeValue();
    return taskMetric;
}