Examples with SupervisorInfo com.alibaba.jstorm.daemon.supervisor.SupervisorInfo used on opensource projects

Example 11 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class ServiceHandler method getClusterInfo.

/**
     * get cluster's summary, it will contain SupervisorSummary and TopologySummary
     *
     * @return ClusterSummary
     */
@Override
public ClusterSummary getClusterInfo() throws TException {
    long start = System.nanoTime();
    try {
        StormClusterState stormClusterState = data.getStormClusterState();
        Map<String, Assignment> assignments = new HashMap<String, Assignment>();
        // get TopologySummary
        List<TopologySummary> topologySummaries = NimbusUtils.getTopologySummary(stormClusterState, assignments);
        // all supervisors
        Map<String, SupervisorInfo> supervisorInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
        // generate SupervisorSummaries
        List<SupervisorSummary> supervisorSummaries = NimbusUtils.mkSupervisorSummaries(supervisorInfos, assignments);
        NimbusSummary nimbusSummary = NimbusUtils.getNimbusSummary(stormClusterState, supervisorSummaries, data);
        return new ClusterSummary(nimbusSummary, supervisorSummaries, topologySummaries);
    } catch (TException e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw e;
    } catch (Exception e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw new TException(e);
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getClusterInfo", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 12 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class TopologyAssign method getFreeSlots.

/**
     * Get free resources
     * 
     * @param supervisorInfos
     * @param stormClusterState
     * @throws Exception
     */
public static void getFreeSlots(Map<String, SupervisorInfo> supervisorInfos, StormClusterState stormClusterState) throws Exception {
    Map<String, Assignment> assignments = Cluster.get_all_assignment(stormClusterState, null);
    for (Entry<String, Assignment> entry : assignments.entrySet()) {
        String topologyId = entry.getKey();
        Assignment assignment = entry.getValue();
        Set<ResourceWorkerSlot> workers = assignment.getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            SupervisorInfo supervisorInfo = supervisorInfos.get(worker.getNodeId());
            if (supervisorInfo == null) {
                // the supervisor is dead
                continue;
            }
            supervisorInfo.getAvailableWorkerPorts().remove(worker.getPort());
        }
    }
}

Example 13 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class TopologyAssign method mkLocalAssignment.

private static Set<ResourceWorkerSlot> mkLocalAssignment(TopologyAssignContext context) throws Exception {
    Set<ResourceWorkerSlot> result = new HashSet<ResourceWorkerSlot>();
    Map<String, SupervisorInfo> cluster = context.getCluster();
    if (cluster.size() != 1)
        throw new RuntimeException();
    SupervisorInfo localSupervisor = null;
    String supervisorId = null;
    for (Entry<String, SupervisorInfo> entry : cluster.entrySet()) {
        supervisorId = entry.getKey();
        localSupervisor = entry.getValue();
    }
    int port = -1;
    if (localSupervisor.getAvailableWorkerPorts().iterator().hasNext()) {
        port = localSupervisor.getAvailableWorkerPorts().iterator().next();
    } else {
        LOG.info(" amount of worker's ports is not enough");
        throw new FailedAssignTopologyException("Failed to make assignment " + ", due to no enough ports");
    }
    ResourceWorkerSlot worker = new ResourceWorkerSlot(supervisorId, port);
    worker.setTasks(new HashSet<Integer>(context.getAllTaskIds()));
    worker.setHostname(localSupervisor.getHostName());
    result.add(worker);
    return result;
}

Example 14 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class TopologyAssign method prepareTopologyAssign.

protected TopologyAssignContext prepareTopologyAssign(TopologyAssignEvent event) throws Exception {
    TopologyAssignContext ret = new TopologyAssignContext();
    String topologyId = event.getTopologyId();
    ret.setTopologyId(topologyId);
    int topoMasterId = nimbusData.getTasksHeartbeat().get(topologyId).get_topologyMasterId();
    ret.setTopologyMasterTaskId(topoMasterId);
    LOG.info("prepareTopologyAssign, topoMasterId={}", topoMasterId);
    Map<Object, Object> nimbusConf = nimbusData.getConf();
    Map<Object, Object> topologyConf = StormConfig.read_nimbus_topology_conf(topologyId, nimbusData.getBlobStore());
    StormTopology rawTopology = StormConfig.read_nimbus_topology_code(topologyId, nimbusData.getBlobStore());
    ret.setRawTopology(rawTopology);
    Map stormConf = new HashMap();
    stormConf.putAll(nimbusConf);
    stormConf.putAll(topologyConf);
    ret.setStormConf(stormConf);
    StormClusterState stormClusterState = nimbusData.getStormClusterState();
    // get all running supervisor, don't need callback to watch supervisor
    Map<String, SupervisorInfo> supInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
    // init all AvailableWorkerPorts
    for (Entry<String, SupervisorInfo> supInfo : supInfos.entrySet()) {
        SupervisorInfo supervisor = supInfo.getValue();
        if (supervisor != null)
            supervisor.setAvailableWorkerPorts(supervisor.getWorkerPorts());
    }
    getAliveSupervsByHb(supInfos, nimbusConf);
    if (supInfos.size() == 0) {
        throw new FailedAssignTopologyException("Failed to make assignment " + topologyId + ", due to no alive supervisor");
    }
    Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
    ret.setTaskToComponent(taskToComponent);
    // get taskids /ZK/tasks/topologyId
    Set<Integer> allTaskIds = taskToComponent.keySet();
    if (allTaskIds == null || allTaskIds.size() == 0) {
        String errMsg = "Failed to get all task ID list from /ZK-dir/tasks/" + topologyId;
        LOG.warn(errMsg);
        throw new IOException(errMsg);
    }
    ret.setAllTaskIds(allTaskIds);
    Set<Integer> aliveTasks = new HashSet<Integer>();
    // unstoppedTasks are tasks which are alive on no supervisor's(dead)
    // machine
    Set<Integer> unstoppedTasks = new HashSet<Integer>();
    Set<Integer> deadTasks = new HashSet<Integer>();
    Set<ResourceWorkerSlot> unstoppedWorkers = new HashSet<ResourceWorkerSlot>();
    Assignment existingAssignment = stormClusterState.assignment_info(topologyId, null);
    if (existingAssignment != null) {
        aliveTasks = getAliveTasks(topologyId, allTaskIds);
        /*
             * Check if the topology master task is alive first since all task 
             * heartbeat info is reported by topology master. 
             * If master is dead, do reassignment for topology master first.
             */
        if (aliveTasks.contains(topoMasterId) == false) {
            ResourceWorkerSlot worker = existingAssignment.getWorkerByTaskId(topoMasterId);
            deadTasks.addAll(worker.getTasks());
            Set<Integer> tempSet = new HashSet<Integer>(allTaskIds);
            tempSet.removeAll(deadTasks);
            aliveTasks.addAll(tempSet);
            aliveTasks.removeAll(deadTasks);
        } else {
            deadTasks.addAll(allTaskIds);
            deadTasks.removeAll(aliveTasks);
        }
        unstoppedTasks = getUnstoppedSlots(aliveTasks, supInfos, existingAssignment);
    }
    ret.setDeadTaskIds(deadTasks);
    ret.setUnstoppedTaskIds(unstoppedTasks);
    // Step 2: get all slots resource, free slots/ alive slots/ unstopped
    // slots
    getFreeSlots(supInfos, stormClusterState);
    ret.setCluster(supInfos);
    if (existingAssignment == null) {
        ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_NEW);
        try {
            AssignmentBak lastAssignment = stormClusterState.assignment_bak(event.getTopologyName());
            if (lastAssignment != null) {
                ret.setOldAssignment(lastAssignment.getAssignment());
            }
        } catch (Exception e) {
            LOG.warn("Fail to get old assignment", e);
        }
    } else {
        ret.setOldAssignment(existingAssignment);
        if (event.isScratch()) {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_REBALANCE);
            ret.setIsReassign(event.isReassign());
            unstoppedWorkers = getUnstoppedWorkers(unstoppedTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        } else {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_MONITOR);
            unstoppedWorkers = getUnstoppedWorkers(aliveTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        }
    }
    return ret;
}

Example 15 with SupervisorInfo

use of com.alibaba.jstorm.daemon.supervisor.SupervisorInfo in project jstorm by alibaba.

the class WorkerScheduler method putAllWorkerToSupervisor.

private void putAllWorkerToSupervisor(List<ResourceWorkerSlot> assignedWorkers, List<SupervisorInfo> supervisors) {
    for (ResourceWorkerSlot worker : assignedWorkers) {
        if (worker.getHostname() != null) {
            for (SupervisorInfo supervisor : supervisors) {
                if (NetWorkUtils.equals(supervisor.getHostName(), worker.getHostname()) && supervisor.getAvailableWorkerPorts().size() > 0) {
                    putWorkerToSupervisor(supervisor, worker);
                    break;
                }
            }
        }
    }
    supervisors = getResAvailSupervisors(supervisors);
    Collections.sort(supervisors, new Comparator<SupervisorInfo>() {

        @Override
        public int compare(SupervisorInfo o1, SupervisorInfo o2) {
            // TODO Auto-generated method stub
            return -NumberUtils.compare(o1.getAvailableWorkerPorts().size(), o2.getAvailableWorkerPorts().size());
        }
    });
    putWorkerToSupervisor(assignedWorkers, supervisors);
}