Examples with ComponentAggregateStats org.apache.storm.generated.ComponentAggregateStats used on opensource projects

Example 1 with ComponentAggregateStats

use of org.apache.storm.generated.ComponentAggregateStats in project storm by apache.

the class TopoWrap method getLogUrls.

public List<ExecutorURL> getLogUrls(final String componentId) throws TException, MalformedURLException {
    ComponentPageInfo componentPageInfo = cluster.getNimbusClient().getComponentPageInfo(id, componentId, null, false);
    Map<String, ComponentAggregateStats> windowToStats = componentPageInfo.get_window_to_stats();
    ComponentAggregateStats allTimeStats = windowToStats.get(":all-time");
    //Long emitted = (Long) allTimeStats.getFieldValue(ComponentAggregateStats._Fields.findByName("emitted"));
    List<ExecutorAggregateStats> execStats = componentPageInfo.get_exec_stats();
    Set<ExecutorURL> urls = new HashSet<>();
    for (ExecutorAggregateStats execStat : execStats) {
        ExecutorSummary execSummary = execStat.get_exec_summary();
        String host = execSummary.get_host();
        int executorPort = execSummary.get_port();
        //http://supervisor2:8000/download/DemoTest-26-1462229009%2F6703%2Fworker.log
        //http://supervisor2:8000/log?file=SlidingWindowCountTest-9-1462388349%2F6703%2Fworker.log
        int logViewerPort = 8000;
        ExecutorURL executorURL = new ExecutorURL(componentId, host, logViewerPort, executorPort, id);
        urls.add(executorURL);
    }
    return new ArrayList<>(urls);
}

Example 2 with ComponentAggregateStats

use of org.apache.storm.generated.ComponentAggregateStats in project storm by apache.

the class Nimbus method getTopologyPageInfo.

@Override
public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyPageInfoCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyPageInfo");
        String topoName = common.topoName;
        IStormClusterState state = stormClusterState;
        int launchTimeSecs = common.launchTimeSecs;
        Assignment assignment = common.assignment;
        Map<List<Integer>, Map<String, Object>> beats = common.beats;
        Map<Integer, String> taskToComp = common.taskToComponent;
        StormTopology topology = common.topology;
        Map<String, Object> topoConf = common.topoConf;
        StormBase base = common.base;
        if (base == null) {
            throw new NotAliveException(topoId);
        }
        Map<WorkerSlot, WorkerResources> workerToResources = getWorkerResourcesForTopology(topoId);
        List<WorkerSummary> workerSummaries = null;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            Map<String, String> nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
            }
            workerSummaries = StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerToResources, includeSys, //this is the topology page, so we know the user is authorized
            true);
        }
        TopologyPageInfo topoPageInfo = StatsUtil.aggTopoExecsStats(topoId, exec2NodePort, taskToComp, beats, topology, window, includeSys, state);
        Map<String, Map<String, Double>> spoutResources = ResourceUtils.getSpoutsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_spout_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(spoutResources, entry.getKey(), topoConf));
        }
        Map<String, Map<String, Double>> boltResources = ResourceUtils.getBoltsResources(topology, topoConf);
        for (Entry<String, ComponentAggregateStats> entry : topoPageInfo.get_id_to_bolt_agg_stats().entrySet()) {
            CommonAggregateStats commonStats = entry.getValue().get_common_stats();
            commonStats.set_resources_map(setResourcesDefaultIfNotSet(boltResources, entry.getKey(), topoConf));
        }
        if (workerSummaries != null) {
            topoPageInfo.set_workers(workerSummaries);
        }
        if (base.is_set_owner()) {
            topoPageInfo.set_owner(base.get_owner());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoPageInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            topoPageInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoPageInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoPageInfo.set_requested_cpu(resources.getRequestedCpu());
            topoPageInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoPageInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoPageInfo.set_assigned_cpu(resources.getAssignedCpu());
        }
        topoPageInfo.set_name(topoName);
        topoPageInfo.set_status(extractStatusStr(base));
        topoPageInfo.set_uptime_secs(Time.deltaSecs(launchTimeSecs));
        topoPageInfo.set_topology_conf(JSONValue.toJSONString(topoConf));
        topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        if (base.is_set_component_debug()) {
            DebugOptions debug = base.get_component_debug().get(topoId);
            if (debug != null) {
                topoPageInfo.set_debug_options(debug);
            }
        }
        return topoPageInfo;
    } catch (Exception e) {
        LOG.warn("Get topo page info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 3 with ComponentAggregateStats

use of org.apache.storm.generated.ComponentAggregateStats in project storm by apache.

the class StatsUtil method postAggregateTopoStats.

public static TopologyPageInfo postAggregateTopoStats(Map task2comp, Map exec2nodePort, Map<String, Object> accData, String topologyId, IStormClusterState clusterState) {
    TopologyPageInfo ret = new TopologyPageInfo(topologyId);
    ret.set_num_tasks(task2comp.size());
    ret.set_num_workers(((Set) getByKey(accData, WORKERS_SET)).size());
    ret.set_num_executors(exec2nodePort != null ? exec2nodePort.size() : 0);
    Map bolt2stats = getMapByKey(accData, BOLT_TO_STATS);
    Map<String, ComponentAggregateStats> aggBolt2stats = new HashMap<>();
    for (Object o : bolt2stats.entrySet()) {
        Map.Entry e = (Map.Entry) o;
        String id = (String) e.getKey();
        Map m = (Map) e.getValue();
        long executed = getByKeyOr0(m, EXECUTED).longValue();
        if (executed > 0) {
            double execLatencyTotal = getByKeyOr0(m, EXEC_LAT_TOTAL).doubleValue();
            putKV(m, EXEC_LATENCY, execLatencyTotal / executed);
            double procLatencyTotal = getByKeyOr0(m, PROC_LAT_TOTAL).doubleValue();
            putKV(m, PROC_LATENCY, procLatencyTotal / executed);
        }
        remove(m, EXEC_LAT_TOTAL);
        remove(m, PROC_LAT_TOTAL);
        putKV(m, "last-error", getLastError(clusterState, topologyId, id));
        aggBolt2stats.put(id, thriftifyBoltAggStats(m));
    }
    Map spout2stats = getMapByKey(accData, SPOUT_TO_STATS);
    Map<String, ComponentAggregateStats> aggSpout2stats = new HashMap<>();
    for (Object o : spout2stats.entrySet()) {
        Map.Entry e = (Map.Entry) o;
        String id = (String) e.getKey();
        Map m = (Map) e.getValue();
        long acked = getByKeyOr0(m, ACKED).longValue();
        if (acked > 0) {
            double compLatencyTotal = getByKeyOr0(m, COMP_LAT_TOTAL).doubleValue();
            putKV(m, COMP_LATENCY, compLatencyTotal / acked);
        }
        remove(m, COMP_LAT_TOTAL);
        putKV(m, "last-error", getLastError(clusterState, topologyId, id));
        aggSpout2stats.put(id, thriftifySpoutAggStats(m));
    }
    TopologyStats topologyStats = new TopologyStats();
    topologyStats.set_window_to_acked(mapKeyStr(getMapByKey(accData, WIN_TO_ACKED)));
    topologyStats.set_window_to_emitted(mapKeyStr(getMapByKey(accData, WIN_TO_EMITTED)));
    topologyStats.set_window_to_failed(mapKeyStr(getMapByKey(accData, WIN_TO_FAILED)));
    topologyStats.set_window_to_transferred(mapKeyStr(getMapByKey(accData, WIN_TO_TRANSFERRED)));
    topologyStats.set_window_to_complete_latencies_ms(computeWeightedAveragesPerWindow(accData, WIN_TO_COMP_LAT_WGT_AVG, WIN_TO_ACKED));
    ret.set_topology_stats(topologyStats);
    ret.set_id_to_spout_agg_stats(aggSpout2stats);
    ret.set_id_to_bolt_agg_stats(aggBolt2stats);
    return ret;
}

Example 4 with ComponentAggregateStats

use of org.apache.storm.generated.ComponentAggregateStats in project storm by apache.

the class StatsUtil method thriftifySpoutAggStats.

private static ComponentAggregateStats thriftifySpoutAggStats(Map m) {
    ComponentAggregateStats stats = new ComponentAggregateStats();
    stats.set_type(ComponentType.SPOUT);
    stats.set_last_error((ErrorInfo) getByKey(m, LAST_ERROR));
    thriftifyCommonAggStats(stats, m);
    SpoutAggregateStats spoutAggStats = new SpoutAggregateStats();
    spoutAggStats.set_complete_latency_ms(getByKeyOr0(m, COMP_LATENCY).doubleValue());
    SpecificAggregateStats specificStats = SpecificAggregateStats.spout(spoutAggStats);
    stats.set_specific_stats(specificStats);
    return stats;
}

Example 5 with ComponentAggregateStats

use of org.apache.storm.generated.ComponentAggregateStats in project storm by apache.

the class StatsUtil method thriftifyBoltAggStats.

private static ComponentAggregateStats thriftifyBoltAggStats(Map m) {
    ComponentAggregateStats stats = new ComponentAggregateStats();
    stats.set_type(ComponentType.BOLT);
    stats.set_last_error((ErrorInfo) getByKey(m, LAST_ERROR));
    thriftifyCommonAggStats(stats, m);
    BoltAggregateStats boltAggStats = new BoltAggregateStats();
    boltAggStats.set_execute_latency_ms(getByKeyOr0(m, EXEC_LATENCY).doubleValue());
    boltAggStats.set_process_latency_ms(getByKeyOr0(m, PROC_LATENCY).doubleValue());
    boltAggStats.set_executed(getByKeyOr0(m, EXECUTED).longValue());
    boltAggStats.set_capacity(getByKeyOr0(m, CAPACITY).doubleValue());
    SpecificAggregateStats specificStats = SpecificAggregateStats.bolt(boltAggStats);
    stats.set_specific_stats(specificStats);
    return stats;
}