Examples with TopologyMetric backtype.storm.generated.TopologyMetric used on opensource projects

Example 1 with TopologyMetric

use of backtype.storm.generated.TopologyMetric in project jstorm by alibaba.

the class ServiceHandler method getTopologyInfo.

/**
     * Get TopologyInfo, it contain all data of the topology running status
     *
     * @return TopologyInfo
     */
@Override
public TopologyInfo getTopologyInfo(String topologyId) throws NotAliveException, TException {
    long start = System.nanoTime();
    StormClusterState stormClusterState = data.getStormClusterState();
    try {
        // get topology's StormBase
        StormBase base = stormClusterState.storm_base(topologyId, null);
        if (base == null) {
            throw new NotAliveException("No topology of " + topologyId);
        }
        Assignment assignment = stormClusterState.assignment_info(topologyId, null);
        if (assignment == null) {
            throw new NotAliveException("No topology of " + topologyId);
        }
        TopologyTaskHbInfo topologyTaskHbInfo = data.getTasksHeartbeat().get(topologyId);
        Map<Integer, TaskHeartbeat> taskHbMap = null;
        if (topologyTaskHbInfo != null)
            taskHbMap = topologyTaskHbInfo.get_taskHbs();
        Map<Integer, TaskInfo> taskInfoMap = Cluster.get_all_taskInfo(stormClusterState, topologyId);
        Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, taskInfoMap);
        Map<Integer, String> taskToType = Cluster.get_all_task_type(stormClusterState, topologyId, taskInfoMap);
        String errorString;
        if (Cluster.is_topology_exist_error(stormClusterState, topologyId)) {
            errorString = "Y";
        } else {
            errorString = "";
        }
        TopologySummary topologySummary = new TopologySummary();
        topologySummary.set_id(topologyId);
        topologySummary.set_name(base.getStormName());
        topologySummary.set_uptimeSecs(TimeUtils.time_delta(base.getLanchTimeSecs()));
        topologySummary.set_status(base.getStatusString());
        topologySummary.set_numTasks(NimbusUtils.getTopologyTaskNum(assignment));
        topologySummary.set_numWorkers(assignment.getWorkers().size());
        topologySummary.set_errorInfo(errorString);
        Map<String, ComponentSummary> componentSummaryMap = new HashMap<String, ComponentSummary>();
        HashMap<String, List<Integer>> componentToTasks = JStormUtils.reverse_map(taskToComponent);
        for (Entry<String, List<Integer>> entry : componentToTasks.entrySet()) {
            String name = entry.getKey();
            List<Integer> taskIds = entry.getValue();
            if (taskIds == null || taskIds.size() == 0) {
                LOG.warn("No task of component " + name);
                continue;
            }
            ComponentSummary componentSummary = new ComponentSummary();
            componentSummaryMap.put(name, componentSummary);
            componentSummary.set_name(name);
            componentSummary.set_type(taskToType.get(taskIds.get(0)));
            componentSummary.set_parallel(taskIds.size());
            componentSummary.set_taskIds(taskIds);
        }
        Map<Integer, TaskSummary> taskSummaryMap = new TreeMap<Integer, TaskSummary>();
        Map<Integer, List<TaskError>> taskErrors = Cluster.get_all_task_errors(stormClusterState, topologyId);
        for (Integer taskId : taskInfoMap.keySet()) {
            TaskSummary taskSummary = new TaskSummary();
            taskSummaryMap.put(taskId, taskSummary);
            taskSummary.set_taskId(taskId);
            if (taskHbMap == null) {
                taskSummary.set_status("Starting");
                taskSummary.set_uptime(0);
            } else {
                TaskHeartbeat hb = taskHbMap.get(taskId);
                if (hb == null) {
                    taskSummary.set_status("Starting");
                    taskSummary.set_uptime(0);
                } else {
                    boolean isInactive = NimbusUtils.isTaskDead(data, topologyId, taskId);
                    if (isInactive)
                        taskSummary.set_status("INACTIVE");
                    else
                        taskSummary.set_status("ACTIVE");
                    taskSummary.set_uptime(hb.get_uptime());
                }
            }
            if (StringUtils.isBlank(errorString)) {
                continue;
            }
            List<TaskError> taskErrorList = taskErrors.get(taskId);
            if (taskErrorList != null && taskErrorList.size() != 0) {
                for (TaskError taskError : taskErrorList) {
                    ErrorInfo errorInfo = new ErrorInfo(taskError.getError(), taskError.getTimSecs(), taskError.getLevel(), taskError.getCode());
                    taskSummary.add_to_errors(errorInfo);
                    String component = taskToComponent.get(taskId);
                    componentSummaryMap.get(component).add_to_errors(errorInfo);
                }
            }
        }
        for (ResourceWorkerSlot workerSlot : assignment.getWorkers()) {
            String hostname = workerSlot.getHostname();
            int port = workerSlot.getPort();
            for (Integer taskId : workerSlot.getTasks()) {
                TaskSummary taskSummary = taskSummaryMap.get(taskId);
                taskSummary.set_host(hostname);
                taskSummary.set_port(port);
            }
        }
        TopologyInfo topologyInfo = new TopologyInfo();
        topologyInfo.set_topology(topologySummary);
        topologyInfo.set_components(JStormUtils.mk_list(componentSummaryMap.values()));
        topologyInfo.set_tasks(JStormUtils.mk_list(taskSummaryMap.values()));
        // return topology metric & component metric only
        List<MetricInfo> tpMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.TOPOLOGY);
        List<MetricInfo> compMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.COMPONENT);
        List<MetricInfo> workerMetricList = data.getMetricCache().getMetricData(topologyId, MetaType.WORKER);
        MetricInfo taskMetric = MetricUtils.mkMetricInfo();
        MetricInfo streamMetric = MetricUtils.mkMetricInfo();
        MetricInfo nettyMetric = MetricUtils.mkMetricInfo();
        MetricInfo tpMetric, compMetric, workerMetric;
        if (tpMetricList == null || tpMetricList.size() == 0) {
            tpMetric = MetricUtils.mkMetricInfo();
        } else {
            // get the last min topology metric
            tpMetric = tpMetricList.get(tpMetricList.size() - 1);
        }
        if (compMetricList == null || compMetricList.size() == 0) {
            compMetric = MetricUtils.mkMetricInfo();
        } else {
            compMetric = compMetricList.get(0);
        }
        if (workerMetricList == null || workerMetricList.size() == 0) {
            workerMetric = MetricUtils.mkMetricInfo();
        } else {
            workerMetric = workerMetricList.get(0);
        }
        TopologyMetric topologyMetrics = new TopologyMetric(tpMetric, compMetric, workerMetric, taskMetric, streamMetric, nettyMetric);
        topologyInfo.set_metrics(topologyMetrics);
        return topologyInfo;
    } catch (TException e) {
        LOG.info("Failed to get topologyInfo " + topologyId, e);
        throw e;
    } catch (Exception e) {
        LOG.info("Failed to get topologyInfo " + topologyId, e);
        throw new TException("Failed to get topologyInfo" + topologyId);
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getTopologyInfo", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 2 with TopologyMetric

use of backtype.storm.generated.TopologyMetric in project jstorm by alibaba.

the class ClusterMetricsContext method getTopologyMetric.

/**
     * get topology metrics, note that only topology & component & worker
     * metrics are returned
     */
public TopologyMetric getTopologyMetric(String topologyId) {
    long start = System.nanoTime();
    try {
        TopologyMetric ret = new TopologyMetric();
        List<MetricInfo> topologyMetrics = metricCache.getMetricData(topologyId, MetaType.TOPOLOGY);
        List<MetricInfo> componentMetrics = metricCache.getMetricData(topologyId, MetaType.COMPONENT);
        List<MetricInfo> workerMetrics = metricCache.getMetricData(topologyId, MetaType.WORKER);
        MetricInfo dummy = MetricUtils.mkMetricInfo();
        if (topologyMetrics.size() > 0) {
            // get the last min topology metric
            ret.set_topologyMetric(topologyMetrics.get(topologyMetrics.size() - 1));
        } else {
            ret.set_topologyMetric(dummy);
        }
        if (componentMetrics.size() > 0) {
            ret.set_componentMetric(componentMetrics.get(0));
        } else {
            ret.set_componentMetric(dummy);
        }
        if (workerMetrics.size() > 0) {
            ret.set_workerMetric(workerMetrics.get(0));
        } else {
            ret.set_workerMetric(dummy);
        }
        ret.set_taskMetric(dummy);
        ret.set_streamMetric(dummy);
        ret.set_nettyMetric(dummy);
        return ret;
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getTopologyMetric", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 3 with TopologyMetric

use of backtype.storm.generated.TopologyMetric in project jstorm by alibaba.

the class MergeEvent method mergeAndUploadClusterMetrics.

private void mergeAndUploadClusterMetrics() {
    TopologyMetricContext clusterContext = context.getClusterTopologyMetricContext();
    TopologyMetric tpMetric = clusterContext.mergeMetrics();
    if (tpMetric == null) {
        tpMetric = MetricUtils.mkTopologyMetric();
        tpMetric.set_topologyMetric(MetricUtils.mkMetricInfo());
    }
    //reset snapshots metric id
    MetricInfo clusterMetrics = tpMetric.get_topologyMetric();
    Map<String, Long> metricName2Id = clusterContext.getMemMeta();
    for (Map.Entry<String, Map<Integer, MetricSnapshot>> entry : clusterMetrics.get_metrics().entrySet()) {
        String metricName = entry.getKey();
        MetricType metricType = MetricUtils.metricType(metricName);
        Long metricId = metricName2Id.get(metricName);
        for (Map.Entry<Integer, MetricSnapshot> metric : entry.getValue().entrySet()) {
            MetricSnapshot snapshot = metric.getValue();
            snapshot.set_metricId(metricId);
            if (metricType == MetricType.HISTOGRAM) {
                snapshot.set_points(new byte[0]);
            }
        //                entry.getValue().put(metric.getKey(), snapshot);
        }
    }
    //fill the unacquired metrics with zero
    long ts = System.currentTimeMillis();
    for (Map.Entry<String, Long> entry : metricName2Id.entrySet()) {
        String name = entry.getKey();
        if (!clusterMetrics.get_metrics().containsKey(name)) {
            Map<Integer, MetricSnapshot> metric = new HashMap<>();
            MetricType type = MetricUtils.metricType(name);
            metric.put(AsmWindow.M1_WINDOW, new MetricSnapshot(entry.getValue(), ts, type.getT()));
            clusterMetrics.put_to_metrics(name, metric);
        }
    }
    //upload to cache
    UpdateEvent.pushEvent(JStormMetrics.CLUSTER_METRIC_KEY, tpMetric);
    LOG.debug("send update event for cluster metrics, size : {}", clusterMetrics.get_metrics_size());
}

Example 4 with TopologyMetric

use of backtype.storm.generated.TopologyMetric in project jstorm by alibaba.

the class JStormMetricsReporter method uploadMetricData.

public void uploadMetricData(WorkerUploadMetrics metrics) {
    if (inTopology) {
        //in Worker, we upload data via netty transport
        if (boltOutput != null) {
            LOG.debug("emit metrics through bolt collector.");
            ((BoltCollector) boltOutput.getDelegate()).emitCtrl(Common.TOPOLOGY_MASTER_METRICS_STREAM_ID, null, new Values(JStormServerUtils.getName(host, port), metrics));
        } else if (spoutOutput != null) {
            LOG.debug("emit metrics through spout collector.");
            ((SpoutCollector) spoutOutput.getDelegate()).emitCtrl(Common.TOPOLOGY_MASTER_METRICS_STREAM_ID, new Values(JStormServerUtils.getName(host, port), metrics), null);
        } else {
            LOG.warn("topology:{}, both spout/bolt collectors are null, don't know what to do...", topologyId);
        }
    } else {
        // in supervisor or nimbus, we upload metric data via thrift
        LOG.debug("emit metrics through nimbus client.");
        TopologyMetric tpMetric = MetricUtils.mkTopologyMetric();
        tpMetric.set_workerMetric(metrics.get_allMetrics());
        //UpdateEvent.pushEvent(topologyId, tpMetric);
        try {
            // push metrics via nimbus client
            if (client == null) {
                LOG.warn("nimbus client is null...");
                client = new NimbusClientWrapper();
                client.init(conf);
            }
            client.getClient().uploadTopologyMetrics(topologyId, tpMetric);
        } catch (Throwable ex) {
            LOG.error("upload metrics error:", ex);
            if (client != null) {
                client.cleanup();
                client = null;
            }
        }
    }
//MetricUtils.logMetrics(metrics.get_allMetrics());
}

Example 5 with TopologyMetric

use of backtype.storm.generated.TopologyMetric in project jstorm by alibaba.

the class MetricUtils method mkTopologyMetric.

public static TopologyMetric mkTopologyMetric() {
    TopologyMetric emptyTopologyMetric = new TopologyMetric();
    emptyTopologyMetric.set_topologyMetric(new MetricInfo());
    emptyTopologyMetric.set_componentMetric(new MetricInfo());
    emptyTopologyMetric.set_workerMetric(new MetricInfo());
    emptyTopologyMetric.set_taskMetric(new MetricInfo());
    emptyTopologyMetric.set_streamMetric(new MetricInfo());
    emptyTopologyMetric.set_nettyMetric(new MetricInfo());
    return emptyTopologyMetric;
}