Examples with ExecutorSummary org.apache.storm.generated.ExecutorSummary used on opensource projects

Example 21 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class StatsUtil method aggregateSpoutStats.

/**
 * aggregate spout stats.
 *
 * @param statsSeq   a seq of ExecutorStats
 * @param includeSys whether to include system streams
 * @return aggregated spout stats: {metric -> win -> global stream id -> value}
 */
public static Map<String, Map> aggregateSpoutStats(List<ExecutorSummary> statsSeq, boolean includeSys) {
    // actually Map<String, Map<String, Map<String, Long/Double>>>
    Map<String, Map> ret = new HashMap<>();
    Map<String, Map<String, Map<String, Long>>> commonStats = aggregateCommonStats(statsSeq);
    // filter sys streams if necessary
    commonStats = preProcessStreamSummary(commonStats, includeSys);
    List<Map<String, Map<String, Long>>> acked = new ArrayList<>();
    List<Map<String, Map<String, Long>>> failed = new ArrayList<>();
    List<Map<String, Map<String, Double>>> completeLatencies = new ArrayList<>();
    for (ExecutorSummary summary : statsSeq) {
        ExecutorStats stats = summary.get_stats();
        acked.add(stats.get_specific().get_spout().get_acked());
        failed.add(stats.get_specific().get_spout().get_failed());
        completeLatencies.add(stats.get_specific().get_spout().get_complete_ms_avg());
    }
    ret.putAll(commonStats);
    ((Map) ret).put(ACKED, aggregateCounts(acked));
    ((Map) ret).put(FAILED, aggregateCounts(failed));
    ((Map) ret).put(COMP_LATENCIES, aggregateAverages(completeLatencies, acked));
    return ret;
}

Example 22 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class LoadMetricsServer method outputMetrics.

private void outputMetrics(Nimbus.Iface client, Collection<String> names) throws Exception {
    Set<String> ids = new HashSet<>();
    HashSet<String> workers = new HashSet<>();
    HashSet<String> hosts = new HashSet<>();
    int executors = 0;
    int uptime = 0;
    long acked = 0;
    long failed = 0;
    double totalLatMs = 0;
    long totalLatCount = 0;
    for (String name : names) {
        TopologyInfo info = client.getTopologyInfoByName(name);
        ids.add(info.get_id());
        @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") TopologyPageInfo tpi = client.getTopologyPageInfo(info.get_id(), ":all-time", false);
        uptime = Math.max(uptime, info.get_uptime_secs());
        for (ExecutorSummary exec : info.get_executors()) {
            hosts.add(exec.get_host());
            workers.add(exec.get_host() + exec.get_port());
            executors++;
            if (exec.get_stats() != null && exec.get_stats().get_specific() != null && exec.get_stats().get_specific().is_set_spout()) {
                SpoutStats stats = exec.get_stats().get_specific().get_spout();
                Map<String, Long> failedMap = stats.get_failed().get(":all-time");
                Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
                if (ackedMap != null) {
                    for (String key : ackedMap.keySet()) {
                        if (failedMap != null) {
                            Long tmp = failedMap.get(key);
                            if (tmp != null) {
                                failed += tmp;
                            }
                        }
                        long ackVal = ackedMap.get(key);
                        acked += ackVal;
                    }
                }
            }
        }
        Double latency = tpi.get_topology_stats().get_window_to_complete_latencies_ms().get(":all-time");
        Long latAcked = tpi.get_topology_stats().get_window_to_acked().get(":all-time");
        if (latency != null && latAcked != null) {
            totalLatCount += latAcked;
            totalLatMs += (latAcked * latency);
        }
    }
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long failedThisTime = failed - prevFailed;
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long ackedThisTime = acked - prevAcked;
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long thisTime = uptime - prevUptime;
    prevUptime = uptime;
    prevAcked = acked;
    prevFailed = failed;
    Histogram copy = new Histogram(3600000000000L, 3);
    ;
    synchronized (histo) {
        copy.add(histo);
        histo.reset();
    }
    long user = userCpu.getAndSet(0);
    long sys = systemCpu.getAndSet(0);
    long gc = gcMs.getAndSet(0);
    long skippedMaxSpout = skippedMaxSpoutMs.getAndSet(0);
    long memBytes = readMemory();
    allCombined.add(new Measurements(uptime, ackedThisTime, thisTime, failedThisTime, copy, user, sys, gc, memBytes, ids, workers.size(), executors, hosts.size(), congested.getAndSet(new ConcurrentHashMap<>()), skippedMaxSpout, totalLatMs / totalLatCount));
    Measurements inWindow = Measurements.combine(allCombined, null, windowLength);
    for (MetricResultsReporter reporter : reporters) {
        reporter.reportWindow(inWindow, allCombined);
    }
}

Example 23 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class UIHelpers method getTopologyWorkers.

/**
 * getTopologyWorkers.
 * @param topologyInfo topologyInfo
 * @param config config
 * @return getTopologyWorkers.
 */
public static Map<String, Object> getTopologyWorkers(TopologyInfo topologyInfo, Map config) {
    List<Map> executorSummaries = new ArrayList();
    for (ExecutorSummary executorSummary : topologyInfo.get_executors()) {
        Map<String, Object> executorSummaryMap = new HashMap();
        executorSummaryMap.put("host", executorSummary.get_host());
        executorSummaryMap.put("port", executorSummary.get_port());
        executorSummaries.add(executorSummaryMap);
    }
    HashSet hashSet = new HashSet();
    hashSet.addAll(executorSummaries);
    executorSummaries.clear();
    executorSummaries.addAll(hashSet);
    Map<String, Object> result = new HashMap();
    result.put("hostPortList", executorSummaries);
    addLogviewerInfo(config, result);
    return result;
}

Example 24 with ExecutorSummary

use of org.apache.storm.generated.ExecutorSummary in project storm by apache.

the class UIHelpers method getSpoutExecutors.

/**
 * getSpoutExecutors.
 * @param executorSummaries executorSummaries
 * @param stormTopology stormTopology
 * @return getSpoutExecutors.
 */
public static Map<String, List<ExecutorSummary>> getSpoutExecutors(List<ExecutorSummary> executorSummaries, StormTopology stormTopology) {
    Map<String, List<ExecutorSummary>> result = new HashMap();
    for (ExecutorSummary executorSummary : executorSummaries) {
        if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()).equals("spout")) {
            List<ExecutorSummary> executorSummaryList = result.getOrDefault(executorSummary.get_component_id(), new ArrayList());
            executorSummaryList.add(executorSummary);
            result.put(executorSummary.get_component_id(), executorSummaryList);
        }
    }
    return result;
}