use of com.alibaba.jstorm.metric.MetricType in project jstorm by alibaba.
the class UpdateEvent method updateClusterMetrics.
//update cluster metrics local cache
private void updateClusterMetrics(String topologyId, TopologyMetric tpMetric) {
if (tpMetric.get_topologyMetric().get_metrics_size() == 0) {
return;
}
MetricInfo topologyMetrics = tpMetric.get_topologyMetric();
// make a new MetricInfo to save the topologyId's metric
MetricInfo clusterMetrics = MetricUtils.mkMetricInfo();
Set<String> metricNames = new HashSet<>();
for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : topologyMetrics.get_metrics().entrySet()) {
String metricName = MetricUtils.topo2clusterName(entry.getKey());
MetricType metricType = MetricUtils.metricType(metricName);
Map<Integer, MetricSnapshot> winData = new HashMap<>();
for (Map.Entry<Integer, MetricSnapshot> entryData : entry.getValue().entrySet()) {
MetricSnapshot snapshot = entryData.getValue().deepCopy();
winData.put(entryData.getKey(), snapshot);
if (metricType == MetricType.HISTOGRAM) {
// reset topology metric points
entryData.getValue().set_points(new byte[0]);
entryData.getValue().set_pointSize(0);
}
}
clusterMetrics.put_to_metrics(metricName, winData);
metricNames.add(metricName);
}
// save to local cache, waiting for merging
TopologyMetricContext clusterTpMetricContext = context.getClusterTopologyMetricContext();
clusterTpMetricContext.addToMemCache(topologyId, clusterMetrics);
context.registerMetrics(JStormMetrics.CLUSTER_METRIC_KEY, metricNames);
}
use of com.alibaba.jstorm.metric.MetricType in project jstorm by alibaba.
the class MetricMeta method getFQN.
public String getFQN() {
MetaType meta = MetaType.parse(metaType);
MetricType metric = MetricType.parse(metricType);
String types = meta.getV() + metric.getV();
if (isWorkerMetric()) {
return MetricUtils.concat2(types, topologyId, host, port, metricGroup, metricName);
}
return MetricUtils.concat2(types, topologyId, component, taskId, streamId, metricGroup, metricName);
}
use of com.alibaba.jstorm.metric.MetricType in project jstorm by alibaba.
the class CheckMetricEvent method run.
@Override
public void run() {
try {
MetricQueryClient metricQueryClient = context.getMetricQueryClient();
int batch = 30;
long start = System.currentTimeMillis() - batch * AsmWindow.M1_WINDOW * TimeUtils.MS_PER_SEC;
long end = System.currentTimeMillis();
List<MetricMeta> toDelete = new ArrayList<>();
for (Pair<MetricMeta, Long> pair : pairs) {
MetricMeta meta = pair.getFirst();
MetricType metricType = MetricType.parse(meta.getMetricType());
Long id1 = meta.getId();
Long id2 = pair.getSecond();
List<Object> data1 = metricQueryClient.getMetricData(id1, metricType, AsmWindow.M1_WINDOW, start, end, batch);
List<Object> data2 = metricQueryClient.getMetricData(id2, metricType, AsmWindow.M1_WINDOW, start, end, batch);
int size1 = data1 != null ? data1.size() : 0;
int size2 = data2 != null ? data2.size() : 0;
LOG.info("check duplicate metric meta for {}, id1:{}, size:{}, id2:{}, size:{}", meta.getMetricName(), id1, size1, id2, size2);
if (size1 > size2) {
// delete id2
meta.setId(id2);
toDelete.add(meta);
tmContext.getMemMeta().put(meta.getFQN(), id1);
} else {
// delete id1
toDelete.add(meta);
tmContext.getMemMeta().put(meta.getFQN(), id2);
}
}
if (toDelete.size() > 0) {
LOG.warn("will delete the following duplicate metric meta\n----------------------------\n");
for (MetricMeta meta : toDelete) {
LOG.warn("metric:{}, id:{}", meta.getFQN(), meta.getId());
}
LOG.info("\n");
metricQueryClient.deleteMeta(toDelete);
}
//context.getMetricCache().putMeta(topologyId, tmContext.getMemMeta());
//context.getStormClusterState().set_topology_metric(topologyId, zkSize - dupMetaList.size());
} catch (Exception ex) {
LOG.error("Error", ex);
}
}
use of com.alibaba.jstorm.metric.MetricType in project jstorm by alibaba.
the class MergeEvent method mergeAndUploadClusterMetrics.
private void mergeAndUploadClusterMetrics() {
TopologyMetricContext clusterContext = context.getClusterTopologyMetricContext();
TopologyMetric tpMetric = clusterContext.mergeMetrics();
if (tpMetric == null) {
tpMetric = MetricUtils.mkTopologyMetric();
tpMetric.set_topologyMetric(MetricUtils.mkMetricInfo());
}
//reset snapshots metric id
MetricInfo clusterMetrics = tpMetric.get_topologyMetric();
Map<String, Long> metricName2Id = clusterContext.getMemMeta();
for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : clusterMetrics.get_metrics().entrySet()) {
String metricName = entry.getKey();
MetricType metricType = MetricUtils.metricType(metricName);
Long metricId = metricName2Id.get(metricName);
for (Map.Entry<Integer, MetricSnapshot> metric : entry.getValue().entrySet()) {
MetricSnapshot snapshot = metric.getValue();
snapshot.set_metricId(metricId);
if (metricType == MetricType.HISTOGRAM) {
snapshot.set_points(new byte[0]);
}
// entry.getValue().put(metric.getKey(), snapshot);
}
}
//fill the unacquired metrics with zero
long ts = System.currentTimeMillis();
for (Map.Entry<String, Long> entry : metricName2Id.entrySet()) {
String name = entry.getKey();
if (!clusterMetrics.get_metrics().containsKey(name)) {
Map<Integer, MetricSnapshot> metric = new HashMap<>();
MetricType type = MetricUtils.metricType(name);
metric.put(AsmWindow.M1_WINDOW, new MetricSnapshot(entry.getValue(), ts, type.getT()));
clusterMetrics.put_to_metrics(name, metric);
}
}
//upload to cache
UpdateEvent.pushEvent(JStormMetrics.CLUSTER_METRIC_KEY, tpMetric);
LOG.debug("send update event for cluster metrics, size : {}", clusterMetrics.get_metrics_size());
}
Aggregations