Examples with SupervisorInfo org.apache.storm.generated.SupervisorInfo used on opensource projects

Example 1 with SupervisorInfo

use of org.apache.storm.generated.SupervisorInfo in project storm by apache.

the class Nimbus method readAllSupervisorDetails.

/**
     * @param superToDeadPorts dead ports on the supervisor
     * @param topologies all of the topologies
     * @param missingAssignmentTopologies topologies that need assignments
     * @return a map: {supervisor-id SupervisorDetails}
     */
private Map<String, SupervisorDetails> readAllSupervisorDetails(Map<String, Set<Long>> superToDeadPorts, Topologies topologies, Collection<String> missingAssignmentTopologies) {
    Map<String, SupervisorDetails> ret = new HashMap<>();
    IStormClusterState state = stormClusterState;
    Map<String, SupervisorInfo> superInfos = state.allSupervisorInfo();
    List<SupervisorDetails> superDetails = new ArrayList<>();
    for (Entry<String, SupervisorInfo> entry : superInfos.entrySet()) {
        SupervisorInfo info = entry.getValue();
        superDetails.add(new SupervisorDetails(entry.getKey(), info.get_meta(), info.get_resources_map()));
    }
    // Note that allSlotsAvailableForScheduling
    // only uses the supervisor-details. The rest of the arguments
    // are there to satisfy the INimbus interface.
    Map<String, Set<Long>> superToPorts = new HashMap<>();
    for (WorkerSlot slot : inimbus.allSlotsAvailableForScheduling(superDetails, topologies, new HashSet<>(missingAssignmentTopologies))) {
        String superId = slot.getNodeId();
        Set<Long> ports = superToPorts.get(superId);
        if (ports == null) {
            ports = new HashSet<>();
            superToPorts.put(superId, ports);
        }
        ports.add((long) slot.getPort());
    }
    for (Entry<String, SupervisorInfo> entry : superInfos.entrySet()) {
        String superId = entry.getKey();
        SupervisorInfo info = entry.getValue();
        String hostname = info.get_hostname();
        // Hide the dead-ports from the all-ports
        // these dead-ports can be reused in next round of assignments
        Set<Long> deadPorts = superToDeadPorts.get(superId);
        Set<Long> allPorts = superToPorts.get(superId);
        if (allPorts == null) {
            allPorts = new HashSet<>();
        } else {
            allPorts = new HashSet<>(allPorts);
        }
        if (deadPorts != null) {
            allPorts.removeAll(deadPorts);
        }
        ret.put(superId, new SupervisorDetails(superId, hostname, info.get_scheduler_meta(), allPorts, info.get_resources_map()));
    }
    return ret;
}

Example 2 with SupervisorInfo

use of org.apache.storm.generated.SupervisorInfo in project storm by apache.

the class Nimbus method getClusterInfoImpl.

private ClusterSummary getClusterInfoImpl() throws Exception {
    IStormClusterState state = stormClusterState;
    Map<String, SupervisorInfo> infos = state.allSupervisorInfo();
    List<SupervisorSummary> summaries = new ArrayList<>(infos.size());
    for (Entry<String, SupervisorInfo> entry : infos.entrySet()) {
        summaries.add(makeSupervisorSummary(entry.getKey(), entry.getValue()));
    }
    int uptime = this.uptime.upTime();
    Map<String, StormBase> bases = state.topologyBases();
    List<NimbusSummary> nimbuses = state.nimbuses();
    //update the isLeader field for each nimbus summary
    NimbusInfo leader = leaderElector.getLeader();
    for (NimbusSummary nimbusSummary : nimbuses) {
        nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs()));
        nimbusSummary.set_isLeader(leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port());
    }
    List<TopologySummary> topologySummaries = new ArrayList<>();
    for (Entry<String, StormBase> entry : bases.entrySet()) {
        StormBase base = entry.getValue();
        if (base == null) {
            continue;
        }
        String topoId = entry.getKey();
        Assignment assignment = state.assignmentInfo(topoId, null);
        int numTasks = 0;
        int numExecutors = 0;
        int numWorkers = 0;
        if (assignment != null && assignment.is_set_executor_node_port()) {
            for (List<Long> ids : assignment.get_executor_node_port().keySet()) {
                numTasks += StormCommon.executorIdToTasks(ids).size();
            }
            numExecutors = assignment.get_executor_node_port_size();
            numWorkers = new HashSet<>(assignment.get_executor_node_port().values()).size();
        }
        TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, numExecutors, numWorkers, Time.deltaSecs(base.get_launch_time_secs()), extractStatusStr(base));
        if (base.is_set_owner()) {
            summary.set_owner(base.get_owner());
        }
        String status = idToSchedStatus.get().get(topoId);
        if (status != null) {
            summary.set_sched_status(status);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            summary.set_requested_memonheap(resources.getRequestedMemOnHeap());
            summary.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            summary.set_requested_cpu(resources.getRequestedCpu());
            summary.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            summary.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            summary.set_assigned_cpu(resources.getAssignedCpu());
        }
        summary.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        topologySummaries.add(summary);
    }
    ClusterSummary ret = new ClusterSummary(summaries, topologySummaries, nimbuses);
    ret.set_nimbus_uptime_secs(uptime);
    return ret;
}

Example 3 with SupervisorInfo

use of org.apache.storm.generated.SupervisorInfo in project storm by apache.

the class Nimbus method getSupervisorPageInfo.

@Override
public SupervisorPageInfo getSupervisorPageInfo(String superId, String host, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getSupervisorPageInfoCalls.mark();
        IStormClusterState state = stormClusterState;
        Map<String, SupervisorInfo> superInfos = state.allSupervisorInfo();
        Map<String, List<String>> hostToSuperId = new HashMap<>();
        for (Entry<String, SupervisorInfo> entry : superInfos.entrySet()) {
            String h = entry.getValue().get_hostname();
            List<String> superIds = hostToSuperId.get(h);
            if (superIds == null) {
                superIds = new ArrayList<>();
                hostToSuperId.put(h, superIds);
            }
            superIds.add(entry.getKey());
        }
        List<String> supervisorIds = null;
        if (superId == null) {
            supervisorIds = hostToSuperId.get(host);
        } else {
            supervisorIds = Arrays.asList(superId);
        }
        SupervisorPageInfo pageInfo = new SupervisorPageInfo();
        Map<String, Assignment> topoToAssignment = state.topologyAssignments();
        for (String sid : supervisorIds) {
            SupervisorInfo info = superInfos.get(sid);
            LOG.info("SIDL {} SI: {} ALL: {}", sid, info, superInfos);
            SupervisorSummary supSum = makeSupervisorSummary(sid, info);
            pageInfo.add_to_supervisor_summaries(supSum);
            List<String> superTopologies = topologiesOnSupervisor(topoToAssignment, sid);
            Set<String> userTopologies = filterAuthorized("getTopology", superTopologies);
            for (String topoId : superTopologies) {
                CommonTopoInfo common = getCommonTopoInfo(topoId, "getSupervisorPageInfo");
                String topoName = common.topoName;
                Assignment assignment = common.assignment;
                Map<List<Integer>, Map<String, Object>> beats = common.beats;
                Map<Integer, String> taskToComp = common.taskToComponent;
                Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
                Map<String, String> nodeToHost;
                if (assignment != null) {
                    Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
                    for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                        NodeInfo ni = entry.getValue();
                        List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                        exec2NodePort.put(entry.getKey(), nodePort);
                    }
                    nodeToHost = assignment.get_node_host();
                } else {
                    nodeToHost = Collections.emptyMap();
                }
                Map<WorkerSlot, WorkerResources> workerResources = getWorkerResourcesForTopology(topoId);
                boolean isAllowed = userTopologies.contains(topoId);
                for (WorkerSummary workerSummary : StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerResources, includeSys, isAllowed, sid)) {
                    pageInfo.add_to_worker_summaries(workerSummary);
                }
            }
        }
        return pageInfo;
    } catch (Exception e) {
        LOG.warn("Get super page info exception. (super id='{}')", superId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 4 with SupervisorInfo

use of org.apache.storm.generated.SupervisorInfo in project storm by apache.

the class SupervisorHeartbeat method run.

@Override
public void run() {
    SupervisorInfo supervisorInfo = buildSupervisorInfo(conf, supervisor);
    stormClusterState.supervisorHeartbeat(supervisorId, supervisorInfo);
}

Example 5 with SupervisorInfo

use of org.apache.storm.generated.SupervisorInfo in project storm by apache.

the class SupervisorHeartbeat method buildSupervisorInfo.

private SupervisorInfo buildSupervisorInfo(Map<String, Object> conf, Supervisor supervisor) {
    SupervisorInfo supervisorInfo = new SupervisorInfo();
    supervisorInfo.set_time_secs(Time.currentTimeSecs());
    supervisorInfo.set_hostname(supervisor.getHostName());
    supervisorInfo.set_assignment_id(supervisor.getAssignmentId());
    List<Long> usedPorts = new ArrayList<>();
    usedPorts.addAll(supervisor.getCurrAssignment().get().keySet());
    supervisorInfo.set_used_ports(usedPorts);
    List metaDatas = (List) supervisor.getiSupervisor().getMetadata();
    List<Long> portList = new ArrayList<>();
    if (metaDatas != null) {
        for (Object data : metaDatas) {
            Integer port = Utils.getInt(data);
            if (port != null)
                portList.add(port.longValue());
        }
    }
    supervisorInfo.set_meta(portList);
    supervisorInfo.set_scheduler_meta((Map<String, String>) conf.get(Config.SUPERVISOR_SCHEDULER_META));
    supervisorInfo.set_uptime_secs(supervisor.getUpTime().upTime());
    supervisorInfo.set_version(supervisor.getStormVersion());
    supervisorInfo.set_resources_map(mkSupervisorCapacities(conf));
    return supervisorInfo;
}