Examples with ClusterSummary org.apache.storm.generated.ClusterSummary used on opensource projects

Example 1 with ClusterSummary

use of org.apache.storm.generated.ClusterSummary in project storm by apache.

the class StormCluster method getSummaries.

public List<TopologySummary> getSummaries() throws TException {
    final ClusterSummary clusterInfo = client.getClusterInfo();
    log.info("Cluster info: " + clusterInfo);
    return clusterInfo.get_topologies();
}

Example 2 with ClusterSummary

use of org.apache.storm.generated.ClusterSummary in project storm by apache.

the class UIHelpers method getClusterSummary.

/**
 * Converts thrift call result into map fit for UI/api.
 * @param clusterSummary Obtained from Nimbus.
 * @param user User Making request
 * @param conf Storm Conf
 * @return Cluster Summary for display on UI/monitoring purposes via API
 */
public static Map<String, Object> getClusterSummary(ClusterSummary clusterSummary, String user, Map<String, Object> conf) {
    Map<String, Object> result = new HashMap();
    if (MEMORIZED_VERSIONS.get() == null) {
        // Races are okay this is just to avoid extra work for each page load.
        NavigableMap<String, IVersionInfo> versionsMap = Utils.getAlternativeVersionsMap(conf);
        List<Map<String, String>> versionList = new ArrayList<>();
        for (Map.Entry<String, IVersionInfo> entry : versionsMap.entrySet()) {
            Map<String, String> single = new HashMap<>(toJsonStruct(entry.getValue()));
            single.put("versionMatch", entry.getKey());
            versionList.add(single);
        }
        MEMORIZED_VERSIONS.set(versionList);
    }
    List<Map<String, String>> versions = MEMORIZED_VERSIONS.get();
    if (!versions.isEmpty()) {
        result.put("alternativeWorkerVersions", versions);
    }
    if (MEMORIZED_FULL_VERSION.get() == null) {
        MEMORIZED_FULL_VERSION.set(toJsonStruct(VersionInfo.OUR_FULL_VERSION));
    }
    result.put("user", user);
    result.put("stormVersion", VersionInfo.getVersion());
    result.put("stormVersionInfo", MEMORIZED_FULL_VERSION.get());
    List<SupervisorSummary> supervisorSummaries = clusterSummary.get_supervisors();
    result.put("supervisors", supervisorSummaries.size());
    result.put("topologies", clusterSummary.get_topologies_size());
    int usedSlots = supervisorSummaries.stream().mapToInt(SupervisorSummary::get_num_used_workers).sum();
    result.put("slotsUsed", usedSlots);
    int totalSlots = supervisorSummaries.stream().mapToInt(SupervisorSummary::get_num_workers).sum();
    result.put("slotsTotal", totalSlots);
    result.put("slotsFree", totalSlots - usedSlots);
    List<TopologySummary> topologySummaries = clusterSummary.get_topologies();
    int totalTasks = topologySummaries.stream().mapToInt(TopologySummary::get_num_tasks).sum();
    result.put("tasksTotal", totalTasks);
    int totalExecutors = topologySummaries.stream().mapToInt(TopologySummary::get_num_executors).sum();
    result.put("executorsTotal", totalExecutors);
    double supervisorTotalMemory = supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(Constants.COMMON_TOTAL_MEMORY_RESOURCE_NAME, x.get_total_resources().get(Config.SUPERVISOR_MEMORY_CAPACITY_MB))).sum();
    result.put("totalMem", supervisorTotalMemory);
    double supervisorTotalCpu = supervisorSummaries.stream().mapToDouble(x -> x.get_total_resources().getOrDefault(Constants.COMMON_CPU_RESOURCE_NAME, x.get_total_resources().get(Config.SUPERVISOR_CPU_CAPACITY))).sum();
    result.put("totalCpu", supervisorTotalCpu);
    double supervisorUsedMemory = supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_mem).sum();
    result.put("availMem", supervisorTotalMemory - supervisorUsedMemory);
    double supervisorUsedCpu = supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_used_cpu).sum();
    result.put("availCpu", supervisorTotalCpu - supervisorUsedCpu);
    result.put("fragmentedMem", supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_fragmented_mem).sum());
    result.put("fragmentedCpu", supervisorSummaries.stream().mapToDouble(SupervisorSummary::get_fragmented_cpu).sum());
    result.put("schedulerDisplayResource", conf.get(DaemonConfig.SCHEDULER_DISPLAY_RESOURCE));
    result.put("memAssignedPercentUtil", supervisorTotalMemory > 0 ? StatsUtil.floatStr((supervisorUsedMemory * 100.0) / supervisorTotalMemory) : "0.0");
    result.put("cpuAssignedPercentUtil", supervisorTotalCpu > 0 ? StatsUtil.floatStr((supervisorUsedCpu * 100.0) / supervisorTotalCpu) : "0.0");
    result.put("bugtracker-url", conf.get(DaemonConfig.UI_PROJECT_BUGTRACKER_URL));
    result.put("central-log-url", conf.get(DaemonConfig.UI_CENTRAL_LOGGING_URL));
    Map<String, Double> usedGenericResources = new HashMap<>();
    Map<String, Double> totalGenericResources = new HashMap<>();
    for (SupervisorSummary ss : supervisorSummaries) {
        usedGenericResources = NormalizedResourceRequest.addResourceMap(usedGenericResources, ss.get_used_generic_resources());
        totalGenericResources = NormalizedResourceRequest.addResourceMap(totalGenericResources, ss.get_total_resources());
    }
    Map<String, Double> availGenericResources = NormalizedResourceRequest.subtractResourceMap(totalGenericResources, usedGenericResources);
    result.put("availGenerics", prettifyGenericResources(availGenericResources));
    result.put("totalGenerics", prettifyGenericResources(totalGenericResources));
    return result;
}

Example 3 with ClusterSummary

use of org.apache.storm.generated.ClusterSummary in project storm by apache.

the class Nimbus method getClusterInfoImpl.

private ClusterSummary getClusterInfoImpl() throws Exception {
    IStormClusterState state = stormClusterState;
    Map<String, SupervisorInfo> infos = state.allSupervisorInfo();
    List<SupervisorSummary> summaries = new ArrayList<>(infos.size());
    for (Entry<String, SupervisorInfo> entry : infos.entrySet()) {
        summaries.add(makeSupervisorSummary(entry.getKey(), entry.getValue()));
    }
    int uptime = this.uptime.upTime();
    List<NimbusSummary> nimbuses = state.nimbuses();
    // update the isLeader field for each nimbus summary
    NimbusInfo leader = leaderElector.getLeader();
    for (NimbusSummary nimbusSummary : nimbuses) {
        nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs()));
        // sometimes Leader election indicates the current nimbus is leader, but the host was recently restarted,
        // and is currently not a leader.
        boolean isLeader = leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port();
        if (isLeader && this.nimbusHostPortInfo.getHost().equals(leader.getHost()) && !this.isLeader()) {
            isLeader = false;
        }
        nimbusSummary.set_isLeader(isLeader);
    }
    List<TopologySummary> topologySummaries = getTopologySummariesImpl();
    ClusterSummary ret = new ClusterSummary(summaries, topologySummaries, nimbuses);
    return ret;
}

Example 4 with ClusterSummary

use of org.apache.storm.generated.ClusterSummary in project storm by apache.

the class TestRebalance method checkTopologyUp.

public boolean checkTopologyUp(String topoName, ILocalCluster cluster) throws TException {
    ClusterSummary sum = cluster.getClusterInfo();
    TopologySummary topoSum = cluster.getTopologySummaryByName(topoName);
    if (topoSum != null) {
        return true;
    }
    return false;
}

Example 5 with ClusterSummary

use of org.apache.storm.generated.ClusterSummary in project storm by apache.

the class MetricsSample method factory.

public static MetricsSample factory(Nimbus.Client client, String topologyName) throws Exception {
    // "************ Sampling Metrics *****************
    ClusterSummary clusterSummary = client.getClusterInfo();
    // get topology info
    TopologySummary topSummary = getTopologySummary(clusterSummary, topologyName);
    int topologyExecutors = topSummary.get_num_executors();
    int topologyWorkers = topSummary.get_num_workers();
    int topologyTasks = topSummary.get_num_tasks();
    TopologyInfo topInfo = client.getTopologyInfo(topSummary.get_id());
    MetricsSample sample = getMetricsSample(topInfo);
    sample.numWorkers = topologyWorkers;
    sample.numExecutors = topologyExecutors;
    sample.numTasks = topologyTasks;
    return sample;
}