Examples with TopologySummary org.apache.storm.generated.TopologySummary used on opensource projects

Example 1 with TopologySummary

use of org.apache.storm.generated.TopologySummary in project storm by apache.

the class StormCluster method killActiveTopologies.

public void killActiveTopologies() throws TException {
    List<TopologySummary> activeTopologies = getActive();
    for (TopologySummary activeTopology : activeTopologies) {
        killSilently(activeTopology.get_name());
    }
    AssertUtil.empty(getActive());
}

Example 2 with TopologySummary

use of org.apache.storm.generated.TopologySummary in project storm by apache.

the class TopoWrap method submitSuccessfully.

public void submitSuccessfully(ImmutableMap<String, Object> config) throws TException {
    submit(config);
    TopologySummary topologySummary = getSummary();
    Assert.assertEquals(topologySummary.get_status().toLowerCase(), "active", "Topology must be active.");
    id = topologySummary.get_id();
}

Example 3 with TopologySummary

use of org.apache.storm.generated.TopologySummary in project storm by apache.

the class Nimbus method getClusterInfoImpl.

private ClusterSummary getClusterInfoImpl() throws Exception {
    IStormClusterState state = stormClusterState;
    Map<String, SupervisorInfo> infos = state.allSupervisorInfo();
    List<SupervisorSummary> summaries = new ArrayList<>(infos.size());
    for (Entry<String, SupervisorInfo> entry : infos.entrySet()) {
        summaries.add(makeSupervisorSummary(entry.getKey(), entry.getValue()));
    }
    int uptime = this.uptime.upTime();
    Map<String, StormBase> bases = state.topologyBases();
    List<NimbusSummary> nimbuses = state.nimbuses();
    //update the isLeader field for each nimbus summary
    NimbusInfo leader = leaderElector.getLeader();
    for (NimbusSummary nimbusSummary : nimbuses) {
        nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs()));
        nimbusSummary.set_isLeader(leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port());
    }
    List<TopologySummary> topologySummaries = new ArrayList<>();
    for (Entry<String, StormBase> entry : bases.entrySet()) {
        StormBase base = entry.getValue();
        if (base == null) {
            continue;
        }
        String topoId = entry.getKey();
        Assignment assignment = state.assignmentInfo(topoId, null);
        int numTasks = 0;
        int numExecutors = 0;
        int numWorkers = 0;
        if (assignment != null && assignment.is_set_executor_node_port()) {
            for (List<Long> ids : assignment.get_executor_node_port().keySet()) {
                numTasks += StormCommon.executorIdToTasks(ids).size();
            }
            numExecutors = assignment.get_executor_node_port_size();
            numWorkers = new HashSet<>(assignment.get_executor_node_port().values()).size();
        }
        TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, numExecutors, numWorkers, Time.deltaSecs(base.get_launch_time_secs()), extractStatusStr(base));
        if (base.is_set_owner()) {
            summary.set_owner(base.get_owner());
        }
        String status = idToSchedStatus.get().get(topoId);
        if (status != null) {
            summary.set_sched_status(status);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            summary.set_requested_memonheap(resources.getRequestedMemOnHeap());
            summary.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            summary.set_requested_cpu(resources.getRequestedCpu());
            summary.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            summary.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            summary.set_assigned_cpu(resources.getAssignedCpu());
        }
        summary.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        topologySummaries.add(summary);
    }
    ClusterSummary ret = new ClusterSummary(summaries, topologySummaries, nimbuses);
    ret.set_nimbus_uptime_secs(uptime);
    return ret;
}

Example 4 with TopologySummary

use of org.apache.storm.generated.TopologySummary in project storm by apache.

the class Nimbus method extractClusterMetrics.

private static List<DataPoint> extractClusterMetrics(ClusterSummary summ) {
    List<DataPoint> ret = new ArrayList<>();
    ret.add(new DataPoint("supervisors", summ.get_supervisors_size()));
    ret.add(new DataPoint("topologies", summ.get_topologies_size()));
    int totalSlots = 0;
    int usedSlots = 0;
    for (SupervisorSummary sup : summ.get_supervisors()) {
        usedSlots += sup.get_num_used_workers();
        totalSlots += sup.get_num_workers();
    }
    ret.add(new DataPoint("slotsTotal", totalSlots));
    ret.add(new DataPoint("slotsUsed", usedSlots));
    ret.add(new DataPoint("slotsFree", totalSlots - usedSlots));
    int totalExecutors = 0;
    int totalTasks = 0;
    for (TopologySummary topo : summ.get_topologies()) {
        totalExecutors += topo.get_num_executors();
        totalTasks += topo.get_num_tasks();
    }
    ret.add(new DataPoint("executorsTotal", totalExecutors));
    ret.add(new DataPoint("tasksTotal", totalTasks));
    return ret;
}

Example 5 with TopologySummary

use of org.apache.storm.generated.TopologySummary in project storm by apache.

the class HdfsSpoutTopology method printMetrics.

static void printMetrics(Nimbus.Client client, String name) throws Exception {
    ClusterSummary summary = client.getClusterInfo();
    String id = null;
    for (TopologySummary ts : summary.get_topologies()) {
        if (name.equals(ts.get_name())) {
            id = ts.get_id();
        }
    }
    if (id == null) {
        throw new Exception("Could not find a topology named " + name);
    }
    TopologyInfo info = client.getTopologyInfo(id);
    int uptime = info.get_uptime_secs();
    long acked = 0;
    long failed = 0;
    double weightedAvgTotal = 0.0;
    for (ExecutorSummary exec : info.get_executors()) {
        if ("spout".equals(exec.get_component_id())) {
            SpoutStats stats = exec.get_stats().get_specific().get_spout();
            Map<String, Long> failedMap = stats.get_failed().get(":all-time");
            Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
            Map<String, Double> avgLatMap = stats.get_complete_ms_avg().get(":all-time");
            for (String key : ackedMap.keySet()) {
                if (failedMap != null) {
                    Long tmp = failedMap.get(key);
                    if (tmp != null) {
                        failed += tmp;
                    }
                }
                long ackVal = ackedMap.get(key);
                double latVal = avgLatMap.get(key) * ackVal;
                acked += ackVal;
                weightedAvgTotal += latVal;
            }
        }
    }
    double avgLatency = weightedAvgTotal / acked;
    System.out.println("uptime: " + uptime + " acked: " + acked + " avgLatency: " + avgLatency + " acked/sec: " + (((double) acked) / uptime + " failed: " + failed));
}