Examples with WorkerSummary org.apache.storm.generated.WorkerSummary used on opensource projects

Example 6 with WorkerSummary

use of org.apache.storm.generated.WorkerSummary in project storm by apache.

the class StatsUtil method aggWorkerStats.

/**
     * aggregate statistics per worker for a topology. Optionally filtering on specific supervisors
     *
     * @param topologyId       topology id
     * @param topology         storm topology
     * @param task2component   a Map of {task id -> component}, note it's a clojure map
     * @param beats            a converted HashMap of executor heartbeats, {executor -> heartbeat}
     * @param exec2hostPort    a Map of {executor -> host+port}, note it's a clojure map
     * @param includeSys       whether to include system streams
     * @param userAuthorized   whether the user is authorized to view topology info
     * @param filterSupervisor if not null, only return WorkerSummaries for that supervisor
     *
     * @return List<WorkerSummary> thrift structures
     */
public static List<WorkerSummary> aggWorkerStats(String stormId, String stormName, Map<Integer, String> task2Component, Map<List<Integer>, Map<String, Object>> beats, Map<List<Long>, List<Object>> exec2NodePort, Map<String, String> nodeHost, Map<WorkerSlot, WorkerResources> worker2Resources, boolean includeSys, boolean userAuthorized, String filterSupervisor) {
    // host,port => WorkerSummary
    HashMap<WorkerSlot, WorkerSummary> workerSummaryMap = new HashMap<>();
    if (exec2NodePort != null) {
        // for each executor -> node+port pair
        for (Map.Entry<List<Long>, List<Object>> execNodePort : exec2NodePort.entrySet()) {
            List<Object> nodePort = execNodePort.getValue();
            String node = (String) nodePort.get(0);
            Long port = (Long) nodePort.get(1);
            String host = nodeHost.get(node);
            WorkerSlot slot = new WorkerSlot(node, port);
            WorkerResources resources = worker2Resources.get(slot);
            if (filterSupervisor == null || node.equals(filterSupervisor)) {
                WorkerSummary ws = workerSummaryMap.get(slot);
                if (ws == null) {
                    ws = new WorkerSummary();
                    ws.set_host(host);
                    ws.set_port(port.intValue());
                    ws.set_supervisor_id(node);
                    ws.set_topology_id(stormId);
                    ws.set_topology_name(stormName);
                    ws.set_num_executors(0);
                    if (resources != null) {
                        ws.set_assigned_memonheap(resources.get_mem_on_heap());
                        ws.set_assigned_memoffheap(resources.get_mem_off_heap());
                        ws.set_assigned_cpu(resources.get_cpu());
                    } else {
                        ws.set_assigned_memonheap(0);
                        ws.set_assigned_memoffheap(0);
                        ws.set_assigned_cpu(0);
                    }
                    ws.set_component_to_num_tasks(new HashMap<String, Long>());
                    workerSummaryMap.put(slot, ws);
                }
                Map<String, Long> componentToNumTasks = ws.get_component_to_num_tasks();
                // gets min/max task pairs (executors): [1 1] [2 3] ...
                List<Long> exec = execNodePort.getKey();
                // get executor heartbeat
                int hbeatSecs = 0;
                if (beats != null) {
                    Map<String, Object> beat = beats.get(convertExecutor(exec));
                    if (beat != null) {
                        Map<String, Object> hbeat = (Map<String, Object>) beat.get("heartbeat");
                        hbeatSecs = hbeat == null ? 0 : (int) hbeat.get("uptime");
                    }
                }
                ws.set_uptime_secs(hbeatSecs);
                ws.set_num_executors(ws.get_num_executors() + 1);
                // get tasks if the user is authorized for this topology
                if (userAuthorized) {
                    int firstTask = exec.get(0).intValue();
                    int lastTask = exec.get(1).intValue();
                    // get per task components
                    for (int task = firstTask; task <= lastTask; task++) {
                        String component = task2Component.get(task);
                        // them in UI, keep going
                        if (!includeSys && Utils.isSystemId(component)) {
                            continue;
                        }
                        // good to go, increment # of tasks this component is being executed on
                        Long counter = componentToNumTasks.get(component);
                        if (counter == null) {
                            counter = new Long(0);
                        }
                        componentToNumTasks.put(component, counter + 1);
                    }
                }
            }
        }
    }
    return new ArrayList<WorkerSummary>(workerSummaryMap.values());
}

Example 7 with WorkerSummary

use of org.apache.storm.generated.WorkerSummary in project storm by apache.

the class TestStatsUtil method aggWorkerStatsFilterSupervisorAndHideSystemComponents.

@Test
public void aggWorkerStatsFilterSupervisorAndHideSystemComponents() {
    makeTopoInfoWithMissingBeats();
    List<WorkerSummary> summaries = checkWorkerStats(false, /*DON'T include sys*/
    true, /*user authorized*/
    "node3");
    WorkerSummary ws = getWorkerSummaryForPort(summaries, 3);
    // hidden sys component
    Assert.assertEquals(1, ws.get_component_to_num_tasks().size());
    Assert.assertEquals(1, ws.get_component_to_num_tasks().get("my-component2").intValue());
    Assert.assertEquals(1, summaries.size());
}

Example 8 with WorkerSummary

use of org.apache.storm.generated.WorkerSummary in project storm by apache.

the class TestStatsUtil method aggWorkerStatsForUnauthorizedUser.

@Test
public void aggWorkerStatsForUnauthorizedUser() {
    makeTopoInfoWithSysWorker();
    List<WorkerSummary> summaries = checkWorkerStats(true, /*include sys (should not matter)*/
    false, /*user NOT authorized*/
    null);
    WorkerSummary ws1 = getWorkerSummaryForPort(summaries, 1);
    WorkerSummary ws2 = getWorkerSummaryForPort(summaries, 2);
    // since we made user not authorized, component map is empty
    Assert.assertEquals(0, ws1.get_component_to_num_tasks().size());
    Assert.assertEquals(0, ws2.get_component_to_num_tasks().size());
    Assert.assertEquals(2, summaries.size());
}

Example 9 with WorkerSummary

use of org.apache.storm.generated.WorkerSummary in project storm by apache.

the class TestStatsUtil method aggWorkerStats.

@Test
public void aggWorkerStats() {
    makeTopoInfo();
    List<WorkerSummary> summaries = checkWorkerStats(true, /*include sys*/
    true, /*user authorized*/
    null);
    WorkerSummary ws = getWorkerSummaryForPort(summaries, 1);
    Assert.assertEquals(1, ws.get_component_to_num_tasks().size());
    Assert.assertEquals(1, ws.get_component_to_num_tasks().get("my-component").intValue());
    Assert.assertEquals(1, summaries.size());
}

Example 10 with WorkerSummary

use of org.apache.storm.generated.WorkerSummary in project storm by apache.

the class TestStatsUtil method aggWorkerStatsWithSystemComponents.

@Test
public void aggWorkerStatsWithSystemComponents() {
    makeTopoInfoWithSysWorker();
    List<WorkerSummary> summaries = checkWorkerStats(true, /*include sys*/
    true, /*user authorized*/
    null);
    WorkerSummary ws = getWorkerSummaryForPort(summaries, 2);
    // since we made sys components visible, the component map has all system components
    Assert.assertEquals(3, ws.get_component_to_num_tasks().size());
    Assert.assertEquals(1, ws.get_component_to_num_tasks().get("__sys1").intValue());
    Assert.assertEquals(1, ws.get_component_to_num_tasks().get("__sys2").intValue());
    Assert.assertEquals(1, ws.get_component_to_num_tasks().get("__sys3").intValue());
    Assert.assertEquals(2, summaries.size());
}