Examples with SpoutStats org.apache.storm.generated.SpoutStats used on opensource projects

Example 6 with SpoutStats

use of org.apache.storm.generated.SpoutStats in project storm by apache.

the class SpoutExecutorStats method renderStats.

@Override
public ExecutorStats renderStats() {
    ExecutorStats ret = new ExecutorStats();
    // common fields
    ret.set_emitted(valueStat(getEmitted()));
    ret.set_transferred(valueStat(getTransferred()));
    ret.set_rate(this.rate);
    // spout stats
    SpoutStats spoutStats = new SpoutStats(valueStat(getAcked()), valueStat(getFailed()), valueStat(completeLatencyStats));
    ret.set_specific(ExecutorSpecificStats.spout(spoutStats));
    return ret;
}

Example 7 with SpoutStats

use of org.apache.storm.generated.SpoutStats in project storm by apache.

the class StatsUtil method thriftifySpecificStats.

private static ExecutorSpecificStats thriftifySpecificStats(Map stats) {
    ExecutorSpecificStats specificStats = new ExecutorSpecificStats();
    String compType = (String) stats.get(TYPE);
    if (ClientStatsUtil.BOLT.equals(compType)) {
        BoltStats boltStats = new BoltStats();
        boltStats.set_acked(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, ACKED), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_executed(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EXECUTED), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_execute_ms_avg(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, EXEC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_failed(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, FAILED), ClientStatsUtil.TO_GSID, TO_STRING));
        boltStats.set_process_ms_avg(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, PROC_LATENCIES), ClientStatsUtil.TO_GSID, TO_STRING));
        specificStats.set_bolt(boltStats);
    } else {
        SpoutStats spoutStats = new SpoutStats();
        spoutStats.set_acked(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, ACKED), TO_STRING, TO_STRING));
        spoutStats.set_failed(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, FAILED), TO_STRING, TO_STRING));
        spoutStats.set_complete_ms_avg(ClientStatsUtil.windowSetConverter(ClientStatsUtil.getMapByKey(stats, COMP_LATENCIES), TO_STRING, TO_STRING));
        specificStats.set_spout(spoutStats);
    }
    return specificStats;
}

Example 8 with SpoutStats

use of org.apache.storm.generated.SpoutStats in project storm by apache.

the class LoadMetricsServer method outputMetrics.

private void outputMetrics(Nimbus.Iface client, Collection<String> names) throws Exception {
    Set<String> ids = new HashSet<>();
    HashSet<String> workers = new HashSet<>();
    HashSet<String> hosts = new HashSet<>();
    int executors = 0;
    int uptime = 0;
    long acked = 0;
    long failed = 0;
    double totalLatMs = 0;
    long totalLatCount = 0;
    for (String name : names) {
        TopologyInfo info = client.getTopologyInfoByName(name);
        ids.add(info.get_id());
        @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") TopologyPageInfo tpi = client.getTopologyPageInfo(info.get_id(), ":all-time", false);
        uptime = Math.max(uptime, info.get_uptime_secs());
        for (ExecutorSummary exec : info.get_executors()) {
            hosts.add(exec.get_host());
            workers.add(exec.get_host() + exec.get_port());
            executors++;
            if (exec.get_stats() != null && exec.get_stats().get_specific() != null && exec.get_stats().get_specific().is_set_spout()) {
                SpoutStats stats = exec.get_stats().get_specific().get_spout();
                Map<String, Long> failedMap = stats.get_failed().get(":all-time");
                Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
                if (ackedMap != null) {
                    for (String key : ackedMap.keySet()) {
                        if (failedMap != null) {
                            Long tmp = failedMap.get(key);
                            if (tmp != null) {
                                failed += tmp;
                            }
                        }
                        long ackVal = ackedMap.get(key);
                        acked += ackVal;
                    }
                }
            }
        }
        Double latency = tpi.get_topology_stats().get_window_to_complete_latencies_ms().get(":all-time");
        Long latAcked = tpi.get_topology_stats().get_window_to_acked().get(":all-time");
        if (latency != null && latAcked != null) {
            totalLatCount += latAcked;
            totalLatMs += (latAcked * latency);
        }
    }
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long failedThisTime = failed - prevFailed;
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long ackedThisTime = acked - prevAcked;
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long thisTime = uptime - prevUptime;
    prevUptime = uptime;
    prevAcked = acked;
    prevFailed = failed;
    Histogram copy = new Histogram(3600000000000L, 3);
    ;
    synchronized (histo) {
        copy.add(histo);
        histo.reset();
    }
    long user = userCpu.getAndSet(0);
    long sys = systemCpu.getAndSet(0);
    long gc = gcMs.getAndSet(0);
    long skippedMaxSpout = skippedMaxSpoutMs.getAndSet(0);
    long memBytes = readMemory();
    allCombined.add(new Measurements(uptime, ackedThisTime, thisTime, failedThisTime, copy, user, sys, gc, memBytes, ids, workers.size(), executors, hosts.size(), congested.getAndSet(new ConcurrentHashMap<>()), skippedMaxSpout, totalLatMs / totalLatCount));
    Measurements inWindow = Measurements.combine(allCombined, null, windowLength);
    for (MetricResultsReporter reporter : reporters) {
        reporter.reportWindow(inWindow, allCombined);
    }
}