Examples with TopologyMetricContext com.alibaba.jstorm.metric.TopologyMetricContext used on opensource projects

Example 6 with TopologyMetricContext

use of com.alibaba.jstorm.metric.TopologyMetricContext in project jstorm by alibaba.

the class TaskStartEvent method run.

@Override
public void run() {
    Assignment assignment = newAssignment;
    TopologyMetricContext metricContext = context.getTopologyMetricContexts().get(topologyId);
    if (metricContext != null) {
        metricContext.setWorkerSet(assignment.getWorkers());
    } else {
        metricContext = new TopologyMetricContext();
        metricContext.setWorkerSet(assignment.getWorkers());
        context.getTopologyMetricContexts().put(topologyId, metricContext);
    }
    context.getMetricUploaderDelegate().sendEvent(context.getClusterName(), this);
}

Example 7 with TopologyMetricContext

use of com.alibaba.jstorm.metric.TopologyMetricContext in project jstorm by alibaba.

the class UpdateEvent method updateClusterMetrics.

// update cluster metrics local cache
private void updateClusterMetrics(String topologyId, TopologyMetric tpMetric) {
    if (tpMetric.get_topologyMetric().get_metrics_size() == 0) {
        return;
    }
    MetricInfo topologyMetrics = tpMetric.get_topologyMetric();
    // make a new MetricInfo to save the topologyId's metric
    MetricInfo clusterMetrics = MetricUtils.mkMetricInfo();
    Set<String> metricNames = new HashSet<>();
    for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : topologyMetrics.get_metrics().entrySet()) {
        String metricName = MetricUtils.topo2clusterName(entry.getKey());
        MetricType metricType = MetricUtils.metricType(metricName);
        Map<Integer, MetricSnapshot> winData = new HashMap<>();
        for (Map.Entry<Integer, MetricSnapshot> entryData : entry.getValue().entrySet()) {
            MetricSnapshot snapshot = entryData.getValue().deepCopy();
            winData.put(entryData.getKey(), snapshot);
            if (metricType == MetricType.HISTOGRAM) {
                // reset topology metric points
                entryData.getValue().set_points(new byte[0]);
                entryData.getValue().set_pointSize(0);
            }
        }
        clusterMetrics.put_to_metrics(metricName, winData);
        metricNames.add(metricName);
    }
    // save to local cache, waiting for merging
    TopologyMetricContext clusterTpMetricContext = context.getClusterTopologyMetricContext();
    clusterTpMetricContext.addToMemCache(topologyId, clusterMetrics);
    context.registerMetrics(JStormMetrics.CLUSTER_METRIC_KEY, metricNames);
}

Example 8 with TopologyMetricContext

use of com.alibaba.jstorm.metric.TopologyMetricContext in project jstorm by alibaba.

the class RefreshEvent method doRefreshTopologies.

/**
 * refresh metric settings of topologies and sync metric meta from local cache
 */
@SuppressWarnings("unchecked")
private void doRefreshTopologies() {
    for (String topology : JStormMetrics.SYS_TOPOLOGIES) {
        if (!context.getTopologyMetricContexts().containsKey(topology)) {
            LOG.info("adding {} to metric context.", topology);
            Map conf = new HashMap();
            if (topology.equals(JStormMetrics.CLUSTER_METRIC_KEY)) {
                // there's no need to consider sample rate when cluster metrics merge
                conf.put(ConfigExtension.TOPOLOGY_METRIC_SAMPLE_RATE, 1.0);
            }
            Set<ResourceWorkerSlot> workerSlot = Sets.newHashSet(new ResourceWorkerSlot());
            TopologyMetricContext metricContext = new TopologyMetricContext(topology, workerSlot, conf);
            context.getTopologyMetricContexts().putIfAbsent(topology, metricContext);
            syncMetaFromCache(topology, context.getTopologyMetricContexts().get(topology));
        }
        // in the first 10 min, sync every 1 min
        if (context.getNimbusData().uptime() < INIT_SYNC_REMOTE_META_TIME_SEC) {
            syncMetaFromRemote(topology, context.getTopologyMetricContexts().get(topology), Lists.newArrayList(MetaType.TOPOLOGY, MetaType.WORKER, MetaType.NIMBUS));
        } else {
            // after that, sync every 5 min
            checkTimeAndSyncMetaFromRemote(topology, context.getTopologyMetricContexts().get(topology), Lists.newArrayList(MetaType.values()));
        }
    }
    Map<String, Assignment> assignMap;
    try {
        assignMap = Cluster.get_all_assignment(context.getStormClusterState(), null);
        for (Entry<String, Assignment> entry : assignMap.entrySet()) {
            String topology = entry.getKey();
            Assignment assignment = entry.getValue();
            TopologyMetricContext metricContext = context.getTopologyMetricContexts().get(topology);
            if (metricContext == null) {
                metricContext = new TopologyMetricContext(assignment.getWorkers());
                metricContext.setTaskNum(NimbusUtils.getTopologyTaskNum(assignment));
                syncMetaFromCache(topology, metricContext);
                LOG.info("adding {} to metric context.", topology);
                context.getTopologyMetricContexts().put(topology, metricContext);
            } else {
                boolean modify = false;
                if (metricContext.getTaskNum() != NimbusUtils.getTopologyTaskNum(assignment)) {
                    modify = true;
                    metricContext.setTaskNum(NimbusUtils.getTopologyTaskNum(assignment));
                }
                if (!assignment.getWorkers().equals(metricContext.getWorkerSet())) {
                    modify = true;
                    metricContext.setWorkerSet(assignment.getWorkers());
                }
                // for normal topologies, only sync topology/component level metrics
                checkTimeAndSyncMetaFromRemote(topology, metricContext, Lists.newArrayList(MetaType.TOPOLOGY, MetaType.COMPONENT));
                // we may need to sync meta when task num/workers change
                metricContext.setSyncMeta(!modify);
            }
        }
    } catch (Exception e1) {
        LOG.warn("Failed to get assignments");
        return;
    }
    List<String> removing = new ArrayList<>();
    for (String topology : context.getTopologyMetricContexts().keySet()) {
        if (!JStormMetrics.SYS_TOPOLOGY_SET.contains(topology) && !assignMap.containsKey(topology)) {
            removing.add(topology);
        }
    }
    for (String topology : removing) {
        LOG.info("removing topology:{}", topology);
        RemoveTopologyEvent.pushEvent(topology);
        syncRemoteTimes.remove(topology);
    }
}

Example 9 with TopologyMetricContext

use of com.alibaba.jstorm.metric.TopologyMetricContext in project jstorm by alibaba.

the class FlushEvent method run.

@Override
public void run() {
    if (!context.getNimbusData().isLeader()) {
        return;
    }
    // if metricUploader is not fully initialized, return directly
    if (!context.isReadyToUpload()) {
        LOG.info("Context Metric Uploader isn't ready");
        return;
    }
    try {
        long start = System.currentTimeMillis();
        for (Map.Entry<String, TopologyMetricContext> entry : context.getTopologyMetricContexts().entrySet()) {
            String topologyId = entry.getKey();
            TopologyMetricContext metricContext = entry.getValue();
            Map<String, Long> cachedMeta = context.getMetricCache().getMeta(topologyId);
            if (cachedMeta == null) {
                cachedMeta = new HashMap<>();
            }
            Map<String, Long> memMeta = metricContext.getMemMeta();
            if (memMeta.size() > cachedMeta.size()) {
                cachedMeta.putAll(memMeta);
            }
            context.getMetricCache().putMeta(topologyId, cachedMeta);
            metricContext.setSyncMeta(false);
            int curSize = cachedMeta.size();
            if (curSize != metricContext.getFlushedMetaNum()) {
                metricContext.setFlushedMetaNum(curSize);
                context.getMetricUploaderDelegate().registerMetrics(context.getClusterName(), topologyId, cachedMeta);
                LOG.info("Flush metric meta, topology:{}, total:{}, cost:{}.", topologyId, curSize, System.currentTimeMillis() - start);
            }
            context.getStormClusterState().set_topology_metric(topologyId, curSize);
        }
    } catch (Exception ex) {
        LOG.error("Error", ex);
    }
}

Example 10 with TopologyMetricContext

use of com.alibaba.jstorm.metric.TopologyMetricContext in project jstorm by alibaba.

the class MergeEvent method mergeAndUploadClusterMetrics.

private void mergeAndUploadClusterMetrics() {
    TopologyMetricContext clusterContext = context.getClusterTopologyMetricContext();
    TopologyMetric tpMetric = clusterContext.mergeMetrics();
    if (tpMetric == null) {
        tpMetric = MetricUtils.mkTopologyMetric();
        tpMetric.set_topologyMetric(MetricUtils.mkMetricInfo());
    }
    // reset snapshots metric id
    MetricInfo clusterMetrics = tpMetric.get_topologyMetric();
    Map<String, Long> metricName2Id = clusterContext.getMemMeta();
    for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : clusterMetrics.get_metrics().entrySet()) {
        String metricName = entry.getKey();
        MetricType metricType = MetricUtils.metricType(metricName);
        Long metricId = metricName2Id.get(metricName);
        for (Map.Entry<Integer, MetricSnapshot> metric : entry.getValue().entrySet()) {
            MetricSnapshot snapshot = metric.getValue();
            snapshot.set_metricId(metricId);
            if (metricType == MetricType.HISTOGRAM) {
                snapshot.set_points(new byte[0]);
            }
        // entry.getValue().put(metric.getKey(), snapshot);
        }
    }
    // fill the unacquired metrics with zero
    long ts = System.currentTimeMillis();
    for (Map.Entry<String, Long> entry : metricName2Id.entrySet()) {
        String name = entry.getKey();
        if (!clusterMetrics.get_metrics().containsKey(name)) {
            Map<Integer, MetricSnapshot> metric = new HashMap<>();
            MetricType type = MetricUtils.metricType(name);
            metric.put(AsmWindow.M1_WINDOW, new MetricSnapshot(entry.getValue(), ts, type.getT()));
            clusterMetrics.put_to_metrics(name, metric);
        }
    }
    // upload to cache
    UpdateEvent.pushEvent(JStormMetrics.CLUSTER_METRIC_KEY, tpMetric);
    LOG.debug("send update event for cluster metrics, size : {}", clusterMetrics.get_metrics_size());
}