Examples with StormClusterState com.alibaba.jstorm.cluster.StormClusterState used on opensource projects

Example 21 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class ServiceHandler method getTopologyTasksToSupervisorIds.

@Override
public Map<Integer, String> getTopologyTasksToSupervisorIds(String topologyName) throws NotAliveException, TException {
    StormClusterState stormClusterState = data.getStormClusterState();
    String topologyId = getTopologyId(topologyName);
    Map<Integer, String> ret = new HashMap<>();
    try {
        Assignment assignment = stormClusterState.assignment_info(topologyId, null);
        Set<ResourceWorkerSlot> workers = assignment.getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            String supervisorId = worker.getNodeId();
            for (Integer task : worker.getTasks()) {
                ret.put(task, supervisorId);
            }
        }
    } catch (Exception ex) {
        LOG.error("Error:", ex);
    }
    return ret;
}

Example 22 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class ServiceHandler method getClusterInfo.

/**
     * get cluster's summary, it will contain SupervisorSummary and TopologySummary
     *
     * @return ClusterSummary
     */
@Override
public ClusterSummary getClusterInfo() throws TException {
    long start = System.nanoTime();
    try {
        StormClusterState stormClusterState = data.getStormClusterState();
        Map<String, Assignment> assignments = new HashMap<String, Assignment>();
        // get TopologySummary
        List<TopologySummary> topologySummaries = NimbusUtils.getTopologySummary(stormClusterState, assignments);
        // all supervisors
        Map<String, SupervisorInfo> supervisorInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
        // generate SupervisorSummaries
        List<SupervisorSummary> supervisorSummaries = NimbusUtils.mkSupervisorSummaries(supervisorInfos, assignments);
        NimbusSummary nimbusSummary = NimbusUtils.getNimbusSummary(stormClusterState, supervisorSummaries, data);
        return new ClusterSummary(nimbusSummary, supervisorSummaries, topologySummaries);
    } catch (TException e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw e;
    } catch (Exception e) {
        LOG.info("Failed to get ClusterSummary ", e);
        throw new TException(e);
    } finally {
        long end = System.nanoTime();
        SimpleJStormMetric.updateNimbusHistogram("getClusterInfo", (end - start) / TimeUtils.NS_PER_US);
    }
}

Example 23 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class TopologyAssign method prepareTopologyAssign.

protected TopologyAssignContext prepareTopologyAssign(TopologyAssignEvent event) throws Exception {
    TopologyAssignContext ret = new TopologyAssignContext();
    String topologyId = event.getTopologyId();
    ret.setTopologyId(topologyId);
    int topoMasterId = nimbusData.getTasksHeartbeat().get(topologyId).get_topologyMasterId();
    ret.setTopologyMasterTaskId(topoMasterId);
    LOG.info("prepareTopologyAssign, topoMasterId={}", topoMasterId);
    Map<Object, Object> nimbusConf = nimbusData.getConf();
    Map<Object, Object> topologyConf = StormConfig.read_nimbus_topology_conf(topologyId, nimbusData.getBlobStore());
    StormTopology rawTopology = StormConfig.read_nimbus_topology_code(topologyId, nimbusData.getBlobStore());
    ret.setRawTopology(rawTopology);
    Map stormConf = new HashMap();
    stormConf.putAll(nimbusConf);
    stormConf.putAll(topologyConf);
    ret.setStormConf(stormConf);
    StormClusterState stormClusterState = nimbusData.getStormClusterState();
    // get all running supervisor, don't need callback to watch supervisor
    Map<String, SupervisorInfo> supInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
    // init all AvailableWorkerPorts
    for (Entry<String, SupervisorInfo> supInfo : supInfos.entrySet()) {
        SupervisorInfo supervisor = supInfo.getValue();
        if (supervisor != null)
            supervisor.setAvailableWorkerPorts(supervisor.getWorkerPorts());
    }
    getAliveSupervsByHb(supInfos, nimbusConf);
    if (supInfos.size() == 0) {
        throw new FailedAssignTopologyException("Failed to make assignment " + topologyId + ", due to no alive supervisor");
    }
    Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
    ret.setTaskToComponent(taskToComponent);
    // get taskids /ZK/tasks/topologyId
    Set<Integer> allTaskIds = taskToComponent.keySet();
    if (allTaskIds == null || allTaskIds.size() == 0) {
        String errMsg = "Failed to get all task ID list from /ZK-dir/tasks/" + topologyId;
        LOG.warn(errMsg);
        throw new IOException(errMsg);
    }
    ret.setAllTaskIds(allTaskIds);
    Set<Integer> aliveTasks = new HashSet<Integer>();
    // unstoppedTasks are tasks which are alive on no supervisor's(dead)
    // machine
    Set<Integer> unstoppedTasks = new HashSet<Integer>();
    Set<Integer> deadTasks = new HashSet<Integer>();
    Set<ResourceWorkerSlot> unstoppedWorkers = new HashSet<ResourceWorkerSlot>();
    Assignment existingAssignment = stormClusterState.assignment_info(topologyId, null);
    if (existingAssignment != null) {
        aliveTasks = getAliveTasks(topologyId, allTaskIds);
        /*
             * Check if the topology master task is alive first since all task 
             * heartbeat info is reported by topology master. 
             * If master is dead, do reassignment for topology master first.
             */
        if (aliveTasks.contains(topoMasterId) == false) {
            ResourceWorkerSlot worker = existingAssignment.getWorkerByTaskId(topoMasterId);
            deadTasks.addAll(worker.getTasks());
            Set<Integer> tempSet = new HashSet<Integer>(allTaskIds);
            tempSet.removeAll(deadTasks);
            aliveTasks.addAll(tempSet);
            aliveTasks.removeAll(deadTasks);
        } else {
            deadTasks.addAll(allTaskIds);
            deadTasks.removeAll(aliveTasks);
        }
        unstoppedTasks = getUnstoppedSlots(aliveTasks, supInfos, existingAssignment);
    }
    ret.setDeadTaskIds(deadTasks);
    ret.setUnstoppedTaskIds(unstoppedTasks);
    // Step 2: get all slots resource, free slots/ alive slots/ unstopped
    // slots
    getFreeSlots(supInfos, stormClusterState);
    ret.setCluster(supInfos);
    if (existingAssignment == null) {
        ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_NEW);
        try {
            AssignmentBak lastAssignment = stormClusterState.assignment_bak(event.getTopologyName());
            if (lastAssignment != null) {
                ret.setOldAssignment(lastAssignment.getAssignment());
            }
        } catch (Exception e) {
            LOG.warn("Fail to get old assignment", e);
        }
    } else {
        ret.setOldAssignment(existingAssignment);
        if (event.isScratch()) {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_REBALANCE);
            ret.setIsReassign(event.isReassign());
            unstoppedWorkers = getUnstoppedWorkers(unstoppedTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        } else {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_MONITOR);
            unstoppedWorkers = getUnstoppedWorkers(aliveTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        }
    }
    return ret;
}

Example 24 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class TopologyAssign method cleanupDisappearedTopology.

/**
     * cleanup the topologies which are not in ZK /topology, but in other place
     * 
     * @throws Exception
     */
public void cleanupDisappearedTopology() throws Exception {
    StormClusterState clusterState = nimbusData.getStormClusterState();
    List<String> active_topologys = clusterState.active_storms();
    if (active_topologys == null) {
        return;
    }
    //        Map<String, List<String>> topoIdToKeys = new HashMap<>();
    Set<String> cleanupIds = get_cleanup_ids(clusterState, active_topologys);
    for (String sysTopology : JStormMetrics.SYS_TOPOLOGIES) {
        cleanupIds.remove(sysTopology);
    }
    for (String topologyId : cleanupIds) {
        LOG.info("Cleaning up " + topologyId);
        clusterState.try_remove_storm(topologyId);
        nimbusData.getTaskHeartbeatsCache().remove(topologyId);
        nimbusData.getTasksHeartbeat().remove(topologyId);
        NimbusUtils.removeTopologyTaskTimeout(nimbusData, topologyId);
        // delete topology files in blobstore
        List<String> deleteKeys = BlobStoreUtils.getKeyListFromId(nimbusData, topologyId);
        BlobStoreUtils.cleanup_keys(deleteKeys, nimbusData.getBlobStore(), nimbusData.getStormClusterState());
    // don't need to delete local dir
    }
}

Example 25 with StormClusterState

use of com.alibaba.jstorm.cluster.StormClusterState in project jstorm by alibaba.

the class TopologyAssign method getTaskStartTimes.

public static Map<Integer, Integer> getTaskStartTimes(TopologyAssignContext context, NimbusData nimbusData, String topologyId, Assignment existingAssignment, Set<ResourceWorkerSlot> workers) throws Exception {
    Map<Integer, Integer> startTimes = new TreeMap<Integer, Integer>();
    if (context.getAssignType() == TopologyAssignContext.ASSIGN_TYPE_NEW) {
        int nowSecs = TimeUtils.current_time_secs();
        for (ResourceWorkerSlot worker : workers) {
            for (Integer changedTaskId : worker.getTasks()) {
                startTimes.put(changedTaskId, nowSecs);
            }
        }
        return startTimes;
    }
    Set<ResourceWorkerSlot> oldWorkers = new HashSet<ResourceWorkerSlot>();
    if (existingAssignment != null) {
        Map<Integer, Integer> taskStartTimeSecs = existingAssignment.getTaskStartTimeSecs();
        if (taskStartTimeSecs != null) {
            startTimes.putAll(taskStartTimeSecs);
        }
        if (existingAssignment.getWorkers() != null) {
            oldWorkers = existingAssignment.getWorkers();
        }
    }
    StormClusterState zkClusterState = nimbusData.getStormClusterState();
    Set<Integer> changedTaskIds = getNewOrChangedTaskIds(oldWorkers, workers);
    int nowSecs = TimeUtils.current_time_secs();
    for (Integer changedTaskId : changedTaskIds) {
        startTimes.put(changedTaskId, nowSecs);
        NimbusUtils.removeTopologyTaskHb(nimbusData, topologyId, changedTaskId);
    }
    Set<Integer> removedTaskIds = getRemovedTaskIds(oldWorkers, workers);
    for (Integer removedTaskId : removedTaskIds) {
        startTimes.remove(removedTaskId);
        NimbusUtils.removeTopologyTaskHb(nimbusData, topologyId, removedTaskId);
    }
    LOG.info("Task assignment has been changed: " + changedTaskIds + ", removed tasks " + removedTaskIds);
    return startTimes;
}