Examples with ExecutorSpecificStats org.apache.storm.generated.ExecutorSpecificStats used on opensource projects

Example 1 with ExecutorSpecificStats

use of org.apache.storm.generated.ExecutorSpecificStats in project storm by apache.

the class MetricsSample method getMetricsSample.

private static MetricsSample getMetricsSample(TopologyInfo topInfo) {
    List<ExecutorSummary> executorSummaries = topInfo.get_executors();
    // totals
    long totalTransferred = 0L;
    long totalEmitted = 0L;
    long totalAcked = 0L;
    long totalFailed = 0L;
    // number of spout executors
    int spoutExecCount = 0;
    double spoutLatencySum = 0.0;
    long spoutTransferred = 0L;
    // Executor summaries
    for (ExecutorSummary executorSummary : executorSummaries) {
        ExecutorStats executorStats = executorSummary.get_stats();
        if (executorStats == null) {
            continue;
        }
        ExecutorSpecificStats executorSpecificStats = executorStats.get_specific();
        if (executorSpecificStats == null) {
            // bail out
            continue;
        }
        // transferred totals
        Map<String, Map<String, Long>> transferred = executorStats.get_transferred();
        Map<String, Long> txMap = transferred.get(":all-time");
        if (txMap == null) {
            continue;
        }
        for (String key : txMap.keySet()) {
            // todo, ignore the master batch coordinator ?
            if (!Utils.isSystemId(key)) {
                Long count = txMap.get(key);
                totalTransferred += count;
                if (executorSpecificStats.is_set_spout()) {
                    spoutTransferred += count;
                }
            }
        }
        // we found a spout
        if (executorSpecificStats.isSet(2)) {
            // spout
            SpoutStats spoutStats = executorSpecificStats.get_spout();
            Map<String, Long> acked = spoutStats.get_acked().get(":all-time");
            if (acked != null) {
                for (String key : acked.keySet()) {
                    totalAcked += acked.get(key);
                }
            }
            Map<String, Long> failed = spoutStats.get_failed().get(":all-time");
            if (failed != null) {
                for (String key : failed.keySet()) {
                    totalFailed += failed.get(key);
                }
            }
            Double total = 0d;
            Map<String, Double> vals = spoutStats.get_complete_ms_avg().get(":all-time");
            if (vals != null) {
                for (String key : vals.keySet()) {
                    total += vals.get(key);
                }
                Double latency = total / vals.size();
                spoutLatencySum += latency;
            }
            spoutExecCount++;
        }
    }
    // end executor summary
    MetricsSample ret = new MetricsSample();
    ret.totalEmitted = totalEmitted;
    ret.totalTransferred = totalTransferred;
    ret.totalAcked = totalAcked;
    ret.totalFailed = totalFailed;
    ret.totalLatency = spoutLatencySum / spoutExecCount;
    long spoutEmitted = 0L;
    ret.spoutEmitted = spoutEmitted;
    ret.spoutTransferred = spoutTransferred;
    ret.sampleTime = System.currentTimeMillis();
    // ret.numSupervisors = clusterSummary.get_supervisors_size();
    ret.numWorkers = 0;
    ret.numExecutors = 0;
    ret.numTasks = 0;
    ret.spoutExecutors = spoutExecCount;
    return ret;
}

Example 2 with ExecutorSpecificStats

use of org.apache.storm.generated.ExecutorSpecificStats in project storm by apache.

the class StatsUtil method thriftifyExecutorStats.

/**
 * Convert Executor stats to thrift data structure.
 * @param stats the stats in the form of a map.
 * @return teh thrift structure for the stats.
 */
public static ExecutorStats thriftifyExecutorStats(Map stats) {
    ExecutorStats ret = new ExecutorStats();
    ExecutorSpecificStats specificStats = thriftifySpecificStats(stats);
    ret.set_specific(specificStats);
    ret.set_emitted(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EMITTED), TO_STRING, TO_STRING));
    ret.set_transferred(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, TRANSFERRED), TO_STRING, TO_STRING));
    ret.set_rate(((Number) stats.get(RATE)).doubleValue());
    return ret;
}

Example 3 with ExecutorSpecificStats

use of org.apache.storm.generated.ExecutorSpecificStats in project storm by apache.

the class StatsUtil method thriftifySpecificStats.

private static ExecutorSpecificStats thriftifySpecificStats(Map stats) {
    ExecutorSpecificStats specificStats = new ExecutorSpecificStats();
    String compType = (String) stats.get(TYPE);
    if (ClientStatsUtil.BOLT.equals(compType)) {
        BoltStats boltStats = new BoltStats();
        boltStats.set_acked(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, ACKED), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_executed(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EXECUTED), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_execute_ms_avg(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EXEC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_failed(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, FAILED), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_process_ms_avg(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, PROC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING));
        specificStats.set_bolt(boltStats);
    } else {
        SpoutStats spoutStats = new SpoutStats();
        spoutStats.set_acked(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, ACKED), TO_STRING, TO_STRING));
        spoutStats.set_failed(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, FAILED), TO_STRING, TO_STRING));
        spoutStats.set_complete_ms_avg(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, COMP_LATENCIES), TO_STRING, TO_STRING));
        specificStats.set_spout(spoutStats);
    }
    return specificStats;
}