Examples with ResourceWorkerSlot com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot used on opensource projects

Example 16 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class TopologyAssign method getFreeSlots.

/**
 * Get free resources
 */
public static void getFreeSlots(Map<String, SupervisorInfo> supervisorInfos, StormClusterState stormClusterState) throws Exception {
    Map<String, Assignment> assignments = Cluster.get_all_assignment(stormClusterState, null);
    for (Entry<String, Assignment> entry : assignments.entrySet()) {
        Assignment assignment = entry.getValue();
        Set<ResourceWorkerSlot> workers = assignment.getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            SupervisorInfo supervisorInfo = supervisorInfos.get(worker.getNodeId());
            if (supervisorInfo == null) {
                // the supervisor is dead
                continue;
            }
            supervisorInfo.getAvailableWorkerPorts().remove(worker.getPort());
        }
    }
}

Example 17 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class TopologyAssign method prepareTopologyAssign.

protected TopologyAssignContext prepareTopologyAssign(TopologyAssignEvent event) throws Exception {
    TopologyAssignContext ret = new TopologyAssignContext();
    String topologyId = event.getTopologyId();
    ret.setTopologyId(topologyId);
    int topoMasterId = nimbusData.getTasksHeartbeat().get(topologyId).get_topologyMasterId();
    ret.setTopologyMasterTaskId(topoMasterId);
    LOG.info("prepareTopologyAssign, topoMasterId={}", topoMasterId);
    Map<Object, Object> nimbusConf = nimbusData.getConf();
    Map<Object, Object> topologyConf = StormConfig.read_nimbus_topology_conf(topologyId, nimbusData.getBlobStore());
    StormTopology rawTopology = StormConfig.read_nimbus_topology_code(topologyId, nimbusData.getBlobStore());
    ret.setRawTopology(rawTopology);
    Map stormConf = new HashMap();
    LOG.info("GET RESERVE_WORKERS from ={}", Utils.readDefaultConfig());
    LOG.info("RESERVE_WORKERS ={}", Utils.readDefaultConfig().get(Config.RESERVE_WORKERS));
    stormConf.put(Config.RESERVE_WORKERS, Utils.readDefaultConfig().get(Config.RESERVE_WORKERS));
    stormConf.putAll(nimbusConf);
    stormConf.putAll(topologyConf);
    ret.setStormConf(stormConf);
    StormClusterState stormClusterState = nimbusData.getStormClusterState();
    // get all running supervisor, don't need callback to watch supervisor
    Map<String, SupervisorInfo> supInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
    // init all AvailableWorkerPorts
    for (Entry<String, SupervisorInfo> supInfo : supInfos.entrySet()) {
        SupervisorInfo supervisor = supInfo.getValue();
        if (supervisor != null)
            supervisor.setAvailableWorkerPorts(supervisor.getWorkerPorts());
    }
    getAliveSupervsByHb(supInfos, nimbusConf);
    if (supInfos.size() == 0) {
        throw new FailedAssignTopologyException("Failed to make assignment " + topologyId + ", due to no alive supervisor");
    }
    Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
    ret.setTaskToComponent(taskToComponent);
    // get taskids /ZK/tasks/topologyId
    Set<Integer> allTaskIds = taskToComponent.keySet();
    if (allTaskIds.size() == 0) {
        String errMsg = "Failed to get all task ID list from /ZK-dir/tasks/" + topologyId;
        LOG.warn(errMsg);
        throw new IOException(errMsg);
    }
    ret.setAllTaskIds(allTaskIds);
    Set<Integer> aliveTasks = new HashSet<>();
    // unstoppedTasks are tasks which are alive on no supervisor's(dead)
    // machine
    Set<Integer> unstoppedTasks = new HashSet<>();
    Set<Integer> deadTasks = new HashSet<>();
    Set<ResourceWorkerSlot> unstoppedWorkers;
    Assignment existingAssignment = stormClusterState.assignment_info(topologyId, null);
    if (existingAssignment != null) {
        /*
             * Check if the topology master task is alive first since all task 
             * heartbeat info is reported by topology master. 
             * If master is dead, do reassignment for topology master first.
             */
        if (NimbusUtils.isTaskDead(nimbusData, topologyId, topoMasterId)) {
            ResourceWorkerSlot tmWorker = existingAssignment.getWorkerByTaskId(topoMasterId);
            deadTasks.addAll(tmWorker.getTasks());
        } else {
            deadTasks = getDeadTasks(topologyId, allTaskIds, existingAssignment.getWorkers());
        }
        aliveTasks.addAll(allTaskIds);
        aliveTasks.removeAll(deadTasks);
        unstoppedTasks = getUnstoppedSlots(aliveTasks, supInfos, existingAssignment);
    }
    ret.setDeadTaskIds(deadTasks);
    ret.setUnstoppedTaskIds(unstoppedTasks);
    // Step 2: get all slots resource, free slots/ alive slots/ unstopped
    // slots
    getFreeSlots(supInfos, stormClusterState);
    ret.setCluster(supInfos);
    if (existingAssignment == null) {
        ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_NEW);
        try {
            AssignmentBak lastAssignment = stormClusterState.assignment_bak(event.getTopologyName());
            if (lastAssignment != null) {
                ret.setOldAssignment(lastAssignment.getAssignment());
            }
        } catch (Exception e) {
            LOG.warn("Fail to get old assignment", e);
        }
    } else {
        ret.setOldAssignment(existingAssignment);
        if (event.isScratch()) {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_REBALANCE);
            ret.setIsReassign(event.isReassign());
            unstoppedWorkers = getUnstoppedWorkers(unstoppedTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        } else {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_MONITOR);
            unstoppedWorkers = getUnstoppedWorkers(aliveTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        }
    }
    return ret;
}

Example 18 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class TopologyAssign method getTopologyNodeHost.

public static Map<String, String> getTopologyNodeHost(Map<String, SupervisorInfo> supervisorMap, Assignment existingAssignment, Set<ResourceWorkerSlot> workers) {
    // the following is that remove unused node from allNodeHost
    Set<String> usedNodes = new HashSet<>();
    for (ResourceWorkerSlot worker : workers) {
        usedNodes.add(worker.getNodeId());
    }
    // map<supervisorId, hostname>
    Map<String, String> allNodeHost = new HashMap<>();
    if (existingAssignment != null) {
        allNodeHost.putAll(existingAssignment.getNodeHost());
    }
    // get alive supervisorMap Map<supervisorId, hostname>
    Map<String, String> nodeHost = SupervisorInfo.getNodeHost(supervisorMap);
    if (nodeHost != null) {
        allNodeHost.putAll(nodeHost);
    }
    Map<String, String> ret = new HashMap<>();
    for (String supervisorId : usedNodes) {
        if (allNodeHost.containsKey(supervisorId)) {
            ret.put(supervisorId, allNodeHost.get(supervisorId));
        } else {
            LOG.warn("Node " + supervisorId + " doesn't in the supervisor list");
        }
    }
    return ret;
}

Example 19 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class TopologyAssign method mkLocalAssignment.

private static Set<ResourceWorkerSlot> mkLocalAssignment(TopologyAssignContext context) throws Exception {
    Set<ResourceWorkerSlot> result = new HashSet<>();
    Map<String, SupervisorInfo> cluster = context.getCluster();
    if (cluster.size() != 1)
        throw new RuntimeException();
    SupervisorInfo localSupervisor = null;
    String supervisorId = null;
    for (Entry<String, SupervisorInfo> entry : cluster.entrySet()) {
        supervisorId = entry.getKey();
        localSupervisor = entry.getValue();
    }
    int port;
    if (localSupervisor.getAvailableWorkerPorts().iterator().hasNext()) {
        port = localSupervisor.getAvailableWorkerPorts().iterator().next();
    } else {
        LOG.info("The amount of worker ports is not enough");
        throw new FailedAssignTopologyException("Failed to make assignment " + ", due to no enough ports");
    }
    ResourceWorkerSlot worker = new ResourceWorkerSlot(supervisorId, port);
    worker.setTasks(new HashSet<>(context.getAllTaskIds()));
    worker.setHostname(localSupervisor.getHostName());
    result.add(worker);
    return result;
}

Example 20 with ResourceWorkerSlot

use of com.alibaba.jstorm.schedule.default_assign.ResourceWorkerSlot in project jstorm by alibaba.

the class TopologyAssign method mkAssignment.

/**
 * make assignments for a topology The nimbus core function, this function has been totally rewrite
 *
 * @throws Exception
 */
public Assignment mkAssignment(TopologyAssignEvent event) throws Exception {
    String topologyId = event.getTopologyId();
    LOG.info("Determining assignment for " + topologyId);
    TopologyAssignContext context = prepareTopologyAssign(event);
    Set<ResourceWorkerSlot> assignments;
    if (!StormConfig.local_mode(nimbusData.getConf())) {
        IToplogyScheduler scheduler = schedulers.get(DEFAULT_SCHEDULER_NAME);
        assignments = scheduler.assignTasks(context);
    } else {
        assignments = mkLocalAssignment(context);
    }
    Assignment assignment = null;
    if (assignments != null && assignments.size() > 0) {
        Map<String, String> nodeHost = getTopologyNodeHost(context.getCluster(), context.getOldAssignment(), assignments);
        Map<Integer, Integer> startTimes = getTaskStartTimes(context, nimbusData, topologyId, context.getOldAssignment(), assignments);
        String codeDir = (String) nimbusData.getConf().get(Config.STORM_LOCAL_DIR);
        assignment = new Assignment(codeDir, assignments, nodeHost, startTimes);
        // the topology binary changed.
        if (event.isScaleTopology()) {
            assignment.setAssignmentType(Assignment.AssignmentType.ScaleTopology);
        }
        StormClusterState stormClusterState = nimbusData.getStormClusterState();
        stormClusterState.set_assignment(topologyId, assignment);
        // update task heartbeat's start time
        NimbusUtils.updateTaskHbStartTime(nimbusData, assignment, topologyId);
        // Update metrics information in ZK when rebalance or reassignment
        // Only update metrics monitor status when creating topology
        // if (context.getAssignType() ==
        // TopologyAssignContext.ASSIGN_TYPE_REBALANCE
        // || context.getAssignType() ==
        // TopologyAssignContext.ASSIGN_TYPE_MONITOR)
        // NimbusUtils.updateMetricsInfo(nimbusData, topologyId, assignment);
        NimbusUtils.updateTopologyTaskTimeout(nimbusData, topologyId);
        LOG.info("Successfully make assignment for topology id " + topologyId + ": " + assignment);
    }
    return assignment;
}