Examples with ExecutorStats org.apache.storm.generated.ExecutorStats used on opensource projects

Example 16 with ExecutorStats

use of org.apache.storm.generated.ExecutorStats in project storm by apache.

the class StatsUtil method aggregateBoltStats.

/**
 * aggregate bolt stats.
 *
 * @param statsSeq   a seq of ExecutorStats
 * @param includeSys whether to include system streams
 * @return aggregated bolt stats: {metric -> win -> global stream id -> value}
 */
public static <T> Map<String, Map> aggregateBoltStats(List<ExecutorSummary> statsSeq, boolean includeSys) {
    Map<String, Map> ret = new HashMap<>();
    Map<String, Map<String, Map<T, Long>>> commonStats = aggregateCommonStats(statsSeq);
    // filter sys streams if necessary
    commonStats = preProcessStreamSummary(commonStats, includeSys);
    List<Map<String, Map<GlobalStreamId, Long>>> acked = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Long>>> failed = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Long>>> executed = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Double>>> processLatencies = new ArrayList<>();
    List<Map<String, Map<GlobalStreamId, Double>>> executeLatencies = new ArrayList<>();
    for (ExecutorSummary summary : statsSeq) {
        ExecutorStats stat = summary.get_stats();
        acked.add(stat.get_specific().get_bolt().get_acked());
        failed.add(stat.get_specific().get_bolt().get_failed());
        executed.add(stat.get_specific().get_bolt().get_executed());
        processLatencies.add(stat.get_specific().get_bolt().get_process_ms_avg());
        executeLatencies.add(stat.get_specific().get_bolt().get_execute_ms_avg());
    }
    mergeMaps(ret, commonStats);
    ((Map) ret).put(ACKED, aggregateCounts(acked));
    ((Map) ret).put(FAILED, aggregateCounts(failed));
    ((Map) ret).put(EXECUTED, aggregateCounts(executed));
    ((Map) ret).put(PROC_LATENCIES, aggregateAverages(processLatencies, acked));
    ((Map) ret).put(EXEC_LATENCIES, aggregateAverages(executeLatencies, executed));
    return ret;
}

Example 17 with ExecutorStats

use of org.apache.storm.generated.ExecutorStats in project storm by apache.

the class StatsUtil method aggregateSpoutStats.

/**
 * aggregate spout stats.
 *
 * @param statsSeq   a seq of ExecutorStats
 * @param includeSys whether to include system streams
 * @return aggregated spout stats: {metric -> win -> global stream id -> value}
 */
public static Map<String, Map> aggregateSpoutStats(List<ExecutorSummary> statsSeq, boolean includeSys) {
    // actually Map<String, Map<String, Map<String, Long/Double>>>
    Map<String, Map> ret = new HashMap<>();
    Map<String, Map<String, Map<String, Long>>> commonStats = aggregateCommonStats(statsSeq);
    // filter sys streams if necessary
    commonStats = preProcessStreamSummary(commonStats, includeSys);
    List<Map<String, Map<String, Long>>> acked = new ArrayList<>();
    List<Map<String, Map<String, Long>>> failed = new ArrayList<>();
    List<Map<String, Map<String, Double>>> completeLatencies = new ArrayList<>();
    for (ExecutorSummary summary : statsSeq) {
        ExecutorStats stats = summary.get_stats();
        acked.add(stats.get_specific().get_spout().get_acked());
        failed.add(stats.get_specific().get_spout().get_failed());
        completeLatencies.add(stats.get_specific().get_spout().get_complete_ms_avg());
    }
    ret.putAll(commonStats);
    ((Map) ret).put(ACKED, aggregateCounts(acked));
    ((Map) ret).put(FAILED, aggregateCounts(failed));
    ((Map) ret).put(COMP_LATENCIES, aggregateAverages(completeLatencies, acked));
    return ret;
}

Example 18 with ExecutorStats

use of org.apache.storm.generated.ExecutorStats in project storm by apache.

the class ClusterUtils method convertExecutorBeats.

/**
 * Ensures that we only return heartbeats for executors assigned to this worker.
 */
public static Map<ExecutorInfo, ExecutorBeat> convertExecutorBeats(List<ExecutorInfo> executors, ClusterWorkerHeartbeat workerHeartbeat) {
    Map<ExecutorInfo, ExecutorBeat> executorWhb = new HashMap<>();
    Map<ExecutorInfo, ExecutorStats> executorStatsMap = workerHeartbeat.get_executor_stats();
    for (ExecutorInfo executor : executors) {
        if (executorStatsMap.containsKey(executor)) {
            int time = workerHeartbeat.get_time_secs();
            int uptime = workerHeartbeat.get_uptime_secs();
            ExecutorStats executorStats = workerHeartbeat.get_executor_stats().get(executor);
            ExecutorBeat executorBeat = new ExecutorBeat(time, uptime, executorStats);
            executorWhb.put(executor, executorBeat);
        }
    }
    return executorWhb;
}