Examples with TopologyInfo org.apache.storm.generated.TopologyInfo used on opensource projects

Example 16 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class MetricsSample method factory.

public static MetricsSample factory(Nimbus.Client client, String topologyName) throws Exception {
    // "************ Sampling Metrics *****************
    ClusterSummary clusterSummary = client.getClusterInfo();
    // get topology info
    TopologySummary topSummary = getTopologySummary(clusterSummary, topologyName);
    int topologyExecutors = topSummary.get_num_executors();
    int topologyWorkers = topSummary.get_num_workers();
    int topologyTasks = topSummary.get_num_tasks();
    TopologyInfo topInfo = client.getTopologyInfo(topSummary.get_id());
    MetricsSample sample = getMetricsSample(topInfo);
    sample.numWorkers = topologyWorkers;
    sample.numExecutors = topologyExecutors;
    sample.numTasks = topologyTasks;
    return sample;
}

Example 17 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class HdfsSpoutTopology method printMetrics.

static void printMetrics(Nimbus.Client client, String name) throws Exception {
    ClusterSummary summary = client.getClusterInfo();
    String id = null;
    for (TopologySummary ts : summary.get_topologies()) {
        if (name.equals(ts.get_name())) {
            id = ts.get_id();
        }
    }
    if (id == null) {
        throw new Exception("Could not find a topology named " + name);
    }
    TopologyInfo info = client.getTopologyInfo(id);
    int uptime = info.get_uptime_secs();
    long acked = 0;
    long failed = 0;
    double weightedAvgTotal = 0.0;
    for (ExecutorSummary exec : info.get_executors()) {
        if ("spout".equals(exec.get_component_id())) {
            SpoutStats stats = exec.get_stats().get_specific().get_spout();
            Map<String, Long> failedMap = stats.get_failed().get(":all-time");
            Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
            Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
            for (String key : ackedMap.keySet()) {
                if (failedMap != null) {
                    Long tmp = failedMap.get(key);
                    if (tmp != null) {
                        failed += tmp;
                    }
                }
                long ackVal = ackedMap.get(key);
                double latVal = avgLatMap.get(key) * ackVal;
                acked += ackVal;
                weightedAvgTotal += latVal;
            }
        }
    }
    double avgLatency = weightedAvgTotal / acked;
    System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " + (((double) acked) / uptime + " failed: " + failed));
}

Example 18 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class Monitor method getComponents.

private HashSet<String> getComponents(Nimbus.Iface client, String topology) throws Exception {
    HashSet<String> components = new HashSet<>();
    GetInfoOptions getInfoOpts = new GetInfoOptions();
    getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
    TopologyInfo info = client.getTopologyInfoByNameWithOpts(topology, getInfoOpts);
    for (ExecutorSummary es : info.get_executors()) {
        components.add(es.get_component_id());
    }
    return components;
}

Example 19 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class Monitor method metrics.

public void metrics(Nimbus.Iface client, long now, MetricsState state) throws Exception {
    long totalStatted = 0;
    int componentParallelism = 0;
    boolean streamFound = false;
    GetInfoOptions getInfoOpts = new GetInfoOptions();
    getInfoOpts.set_num_err_choice(NumErrorsChoice.NONE);
    TopologyInfo info = client.getTopologyInfoByNameWithOpts(topology, getInfoOpts);
    for (ExecutorSummary es : info.get_executors()) {
        if (component.equals(es.get_component_id())) {
            componentParallelism++;
            ExecutorStats stats = es.get_stats();
            if (stats != null) {
                Map<String, Map<String, Long>> statted = WATCH_EMITTED.equals(watch) ? stats.get_emitted() : stats.get_transferred();
                if (statted != null) {
                    Map<String, Long> e2 = statted.get(":all-time");
                    if (e2 != null) {
                        Long stream = e2.get(this.stream);
                        if (stream != null) {
                            streamFound = true;
                            totalStatted += stream;
                        }
                    }
                }
            }
        }
    }
    if (componentParallelism <= 0) {
        HashSet<String> components = getComponents(client, topology);
        System.out.println("Available components for " + topology + " :");
        System.out.println("------------------");
        for (String comp : components) {
            System.out.println(comp);
        }
        System.out.println("------------------");
        throw new IllegalArgumentException("component: " + component + " not found");
    }
    if (!streamFound) {
        throw new IllegalArgumentException("stream: " + stream + " not found");
    }
    long timeDelta = now - state.getLastTime();
    long stattedDelta = totalStatted - state.getLastStatted();
    state.setLastTime(now);
    state.setLastStatted(totalStatted);
    double throughput = (stattedDelta == 0 || timeDelta == 0) ? 0.0 : ((double) stattedDelta / (double) timeDelta);
    System.out.println(topology + "\t" + component + "\t" + componentParallelism + "\t" + stream + "\t" + timeDelta + "\t" + stattedDelta + "\t" + throughput);
}

Example 20 with TopologyInfo

use of org.apache.storm.generated.TopologyInfo in project storm by apache.

the class LoadMetricsServer method outputMetrics.

private void outputMetrics(Nimbus.Iface client, Collection<String> names) throws Exception {
    Set<String> ids = new HashSet<>();
    HashSet<String> workers = new HashSet<>();
    HashSet<String> hosts = new HashSet<>();
    int executors = 0;
    int uptime = 0;
    long acked = 0;
    long failed = 0;
    double totalLatMs = 0;
    long totalLatCount = 0;
    for (String name : names) {
        TopologyInfo info = client.getTopologyInfoByName(name);
        ids.add(info.get_id());
        @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") TopologyPageInfo tpi = client.getTopologyPageInfo(info.get_id(), ":all-time", false);
        uptime = Math.max(uptime, info.get_uptime_secs());
        for (ExecutorSummary exec : info.get_executors()) {
            hosts.add(exec.get_host());
            workers.add(exec.get_host() + exec.get_port());
            executors++;
            if (exec.get_stats() != null && exec.get_stats().get_specific() != null && exec.get_stats().get_specific().is_set_spout()) {
                SpoutStats stats = exec.get_stats().get_specific().get_spout();
                Map<String, Long> failedMap = stats.get_failed().get(":all-time");
                Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
                if (ackedMap != null) {
                    for (String key : ackedMap.keySet()) {
                        if (failedMap != null) {
                            Long tmp = failedMap.get(key);
                            if (tmp != null) {
                                failed += tmp;
                            }
                        }
                        long ackVal = ackedMap.get(key);
                        acked += ackVal;
                    }
                }
            }
        }
        Double latency = tpi.get_topology_stats().get_window_to_complete_latencies_ms().get(":all-time");
        Long latAcked = tpi.get_topology_stats().get_window_to_acked().get(":all-time");
        if (latency != null && latAcked != null) {
            totalLatCount += latAcked;
            totalLatMs += (latAcked * latency);
        }
    }
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long failedThisTime = failed - prevFailed;
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long ackedThisTime = acked - prevAcked;
    @SuppressWarnings("checkstyle:VariableDeclarationUsageDistance") long thisTime = uptime - prevUptime;
    prevUptime = uptime;
    prevAcked = acked;
    prevFailed = failed;
    Histogram copy = new Histogram(3600000000000L, 3);
    ;
    synchronized (histo) {
        copy.add(histo);
        histo.reset();
    }
    long user = userCpu.getAndSet(0);
    long sys = systemCpu.getAndSet(0);
    long gc = gcMs.getAndSet(0);
    long skippedMaxSpout = skippedMaxSpoutMs.getAndSet(0);
    long memBytes = readMemory();
    allCombined.add(new Measurements(uptime, ackedThisTime, thisTime, failedThisTime, copy, user, sys, gc, memBytes, ids, workers.size(), executors, hosts.size(), congested.getAndSet(new ConcurrentHashMap<>()), skippedMaxSpout, totalLatMs / totalLatCount));
    Measurements inWindow = Measurements.combine(allCombined, null, windowLength);
    for (MetricResultsReporter reporter : reporters) {
        reporter.reportWindow(inWindow, allCombined);
    }
}