Examples with TopologyAssignContext com.alibaba.jstorm.schedule.TopologyAssignContext used on opensource projects

Example 1 with TopologyAssignContext

use of com.alibaba.jstorm.schedule.TopologyAssignContext in project jstorm by alibaba.

the class TopologyAssign method prepareTopologyAssign.

protected TopologyAssignContext prepareTopologyAssign(TopologyAssignEvent event) throws Exception {
    TopologyAssignContext ret = new TopologyAssignContext();
    String topologyId = event.getTopologyId();
    ret.setTopologyId(topologyId);
    int topoMasterId = nimbusData.getTasksHeartbeat().get(topologyId).get_topologyMasterId();
    ret.setTopologyMasterTaskId(topoMasterId);
    LOG.info("prepareTopologyAssign, topoMasterId={}", topoMasterId);
    Map<Object, Object> nimbusConf = nimbusData.getConf();
    Map<Object, Object> topologyConf = StormConfig.read_nimbus_topology_conf(topologyId, nimbusData.getBlobStore());
    StormTopology rawTopology = StormConfig.read_nimbus_topology_code(topologyId, nimbusData.getBlobStore());
    ret.setRawTopology(rawTopology);
    Map stormConf = new HashMap();
    LOG.info("GET RESERVE_WORKERS from ={}", Utils.readDefaultConfig());
    LOG.info("RESERVE_WORKERS ={}", Utils.readDefaultConfig().get(Config.RESERVE_WORKERS));
    stormConf.put(Config.RESERVE_WORKERS, Utils.readDefaultConfig().get(Config.RESERVE_WORKERS));
    stormConf.putAll(nimbusConf);
    stormConf.putAll(topologyConf);
    ret.setStormConf(stormConf);
    StormClusterState stormClusterState = nimbusData.getStormClusterState();
    // get all running supervisor, don't need callback to watch supervisor
    Map<String, SupervisorInfo> supInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
    // init all AvailableWorkerPorts
    for (Entry<String, SupervisorInfo> supInfo : supInfos.entrySet()) {
        SupervisorInfo supervisor = supInfo.getValue();
        if (supervisor != null)
            supervisor.setAvailableWorkerPorts(supervisor.getWorkerPorts());
    }
    getAliveSupervsByHb(supInfos, nimbusConf);
    if (supInfos.size() == 0) {
        throw new FailedAssignTopologyException("Failed to make assignment " + topologyId + ", due to no alive supervisor");
    }
    Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
    ret.setTaskToComponent(taskToComponent);
    // get taskids /ZK/tasks/topologyId
    Set<Integer> allTaskIds = taskToComponent.keySet();
    if (allTaskIds.size() == 0) {
        String errMsg = "Failed to get all task ID list from /ZK-dir/tasks/" + topologyId;
        LOG.warn(errMsg);
        throw new IOException(errMsg);
    }
    ret.setAllTaskIds(allTaskIds);
    Set<Integer> aliveTasks = new HashSet<>();
    // unstoppedTasks are tasks which are alive on no supervisor's(dead)
    // machine
    Set<Integer> unstoppedTasks = new HashSet<>();
    Set<Integer> deadTasks = new HashSet<>();
    Set<ResourceWorkerSlot> unstoppedWorkers;
    Assignment existingAssignment = stormClusterState.assignment_info(topologyId, null);
    if (existingAssignment != null) {
        /*
             * Check if the topology master task is alive first since all task 
             * heartbeat info is reported by topology master. 
             * If master is dead, do reassignment for topology master first.
             */
        if (NimbusUtils.isTaskDead(nimbusData, topologyId, topoMasterId)) {
            ResourceWorkerSlot tmWorker = existingAssignment.getWorkerByTaskId(topoMasterId);
            deadTasks.addAll(tmWorker.getTasks());
        } else {
            deadTasks = getDeadTasks(topologyId, allTaskIds, existingAssignment.getWorkers());
        }
        aliveTasks.addAll(allTaskIds);
        aliveTasks.removeAll(deadTasks);
        unstoppedTasks = getUnstoppedSlots(aliveTasks, supInfos, existingAssignment);
    }
    ret.setDeadTaskIds(deadTasks);
    ret.setUnstoppedTaskIds(unstoppedTasks);
    // Step 2: get all slots resource, free slots/ alive slots/ unstopped
    // slots
    getFreeSlots(supInfos, stormClusterState);
    ret.setCluster(supInfos);
    if (existingAssignment == null) {
        ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_NEW);
        try {
            AssignmentBak lastAssignment = stormClusterState.assignment_bak(event.getTopologyName());
            if (lastAssignment != null) {
                ret.setOldAssignment(lastAssignment.getAssignment());
            }
        } catch (Exception e) {
            LOG.warn("Fail to get old assignment", e);
        }
    } else {
        ret.setOldAssignment(existingAssignment);
        if (event.isScratch()) {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_REBALANCE);
            ret.setIsReassign(event.isReassign());
            unstoppedWorkers = getUnstoppedWorkers(unstoppedTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        } else {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_MONITOR);
            unstoppedWorkers = getUnstoppedWorkers(aliveTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        }
    }
    return ret;
}

Example 2 with TopologyAssignContext

use of com.alibaba.jstorm.schedule.TopologyAssignContext in project jstorm by alibaba.

the class TopologyAssign method mkAssignment.

/**
 * make assignments for a topology The nimbus core function, this function has been totally rewrite
 *
 * @throws Exception
 */
public Assignment mkAssignment(TopologyAssignEvent event) throws Exception {
    String topologyId = event.getTopologyId();
    LOG.info("Determining assignment for " + topologyId);
    TopologyAssignContext context = prepareTopologyAssign(event);
    Set<ResourceWorkerSlot> assignments;
    if (!StormConfig.local_mode(nimbusData.getConf())) {
        IToplogyScheduler scheduler = schedulers.get(DEFAULT_SCHEDULER_NAME);
        assignments = scheduler.assignTasks(context);
    } else {
        assignments = mkLocalAssignment(context);
    }
    Assignment assignment = null;
    if (assignments != null && assignments.size() > 0) {
        Map<String, String> nodeHost = getTopologyNodeHost(context.getCluster(), context.getOldAssignment(), assignments);
        Map<Integer, Integer> startTimes = getTaskStartTimes(context, nimbusData, topologyId, context.getOldAssignment(), assignments);
        String codeDir = (String) nimbusData.getConf().get(Config.STORM_LOCAL_DIR);
        assignment = new Assignment(codeDir, assignments, nodeHost, startTimes);
        // the topology binary changed.
        if (event.isScaleTopology()) {
            assignment.setAssignmentType(Assignment.AssignmentType.ScaleTopology);
        }
        StormClusterState stormClusterState = nimbusData.getStormClusterState();
        stormClusterState.set_assignment(topologyId, assignment);
        // update task heartbeat's start time
        NimbusUtils.updateTaskHbStartTime(nimbusData, assignment, topologyId);
        // Update metrics information in ZK when rebalance or reassignment
        // Only update metrics monitor status when creating topology
        // if (context.getAssignType() ==
        // TopologyAssignContext.ASSIGN_TYPE_REBALANCE
        // || context.getAssignType() ==
        // TopologyAssignContext.ASSIGN_TYPE_MONITOR)
        // NimbusUtils.updateMetricsInfo(nimbusData, topologyId, assignment);
        NimbusUtils.updateTopologyTaskTimeout(nimbusData, topologyId);
        LOG.info("Successfully make assignment for topology id " + topologyId + ": " + assignment);
    }
    return assignment;
}