Examples with Assignment com.alibaba.jstorm.schedule.Assignment used on opensource projects

Example 26 with Assignment

use of com.alibaba.jstorm.schedule.Assignment in project jstorm by alibaba.

the class SyncSupervisorEvent method getLocalAssign.

/**
 * a port must be assigned to a topology
 *
 * @return map: [port,LocalAssignment]
 */
private Map<Integer, LocalAssignment> getLocalAssign(StormClusterState stormClusterState, String supervisorId, Map<String, Assignment> assignments) throws Exception {
    Map<Integer, LocalAssignment> portToAssignment = new HashMap<>();
    for (Entry<String, Assignment> assignEntry : assignments.entrySet()) {
        String topologyId = assignEntry.getKey();
        Assignment assignment = assignEntry.getValue();
        Map<Integer, LocalAssignment> portTasks = readMyTasks(stormClusterState, topologyId, supervisorId, assignment);
        if (portTasks == null) {
            continue;
        }
        // a port must be assigned to one assignment
        for (Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
            Integer port = entry.getKey();
            LocalAssignment la = entry.getValue();
            if (!portToAssignment.containsKey(port)) {
                portToAssignment.put(port, la);
            } else {
                throw new RuntimeException("Should not have multiple topologies assigned to one port");
            }
        }
    }
    return portToAssignment;
}

Example 27 with Assignment

use of com.alibaba.jstorm.schedule.Assignment in project jstorm by alibaba.

the class SyncSupervisorEvent method getTopologyCodeLocations.

/**
 * get master code dir for each topology
 *
 * @return Map: [topologyId, master-code-dir] from zookeeper
 */
public static Map<String, String> getTopologyCodeLocations(Map<String, Assignment> assignments, String supervisorId) throws Exception {
    Map<String, String> rtn = new HashMap<>();
    for (Entry<String, Assignment> entry : assignments.entrySet()) {
        String topologyId = entry.getKey();
        Assignment assignmentInfo = entry.getValue();
        Set<ResourceWorkerSlot> workers = assignmentInfo.getWorkers();
        for (ResourceWorkerSlot worker : workers) {
            String node = worker.getNodeId();
            if (supervisorId.equals(node)) {
                rtn.put(topologyId, assignmentInfo.getMasterCodeDir());
                break;
            }
        }
    }
    return rtn;
}

Example 28 with Assignment

use of com.alibaba.jstorm.schedule.Assignment in project jstorm by alibaba.

the class SyncSupervisorEvent method run.

@SuppressWarnings("unchecked")
@Override
public void run() {
    LOG.debug("Synchronizing supervisor, interval (sec): " + TimeUtils.time_delta(lastTime));
    lastTime = TimeUtils.current_time_secs();
    // make sure that the status is the same for each execution of syncsupervisor
    HealthStatus healthStatus = heartbeat.getHealthStatus();
    try {
        RunnableCallback syncCallback = new EventManagerZkPusher(this, syncSupEventManager);
        Map<String, Integer> assignmentVersion = (Map<String, Integer>) localState.get(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION);
        if (assignmentVersion == null) {
            assignmentVersion = new HashMap<>();
        }
        Map<String, Assignment> assignments = (Map<String, Assignment>) localState.get(Common.LS_LOCAl_ZK_ASSIGNMENTS);
        if (assignments == null) {
            assignments = new HashMap<>();
        }
        LOG.debug("get local assignments  " + assignments);
        LOG.debug("get local assignments version " + assignmentVersion);
        /**
         * Step 1: get all assignments and add assignment watchers for /ZK-dir/assignment
         */
        if (healthStatus.isMoreSeriousThan(HealthStatus.ERROR)) {
            // if status is panic or error, clear all assignments and kill all workers
            assignmentVersion.clear();
            assignments.clear();
            LOG.warn("Supervisor machine check status: " + healthStatus + ", killing all workers.");
        } else {
            getAllAssignments(assignmentVersion, assignments, syncCallback);
        }
        LOG.debug("Get all assignments " + assignments);
        /**
         * Step 2: get topology id list from STORM-LOCAL-DIR/supervisor/stormdist/
         */
        List<String> downloadedTopologyIds = StormConfig.get_supervisor_toplogy_list(conf);
        LOG.debug("Downloaded storm ids: " + downloadedTopologyIds);
        /**
         * Step 3: get <port,LocalAssignments> from ZK local node's assignment
         */
        Map<Integer, LocalAssignment> zkAssignment = getLocalAssign(stormClusterState, supervisorId, assignments);
        Map<Integer, LocalAssignment> localAssignment;
        /**
         * Step 4: writer local assignment to LocalState
         */
        try {
            LOG.debug("Writing local assignment " + zkAssignment);
            localAssignment = (Map<Integer, LocalAssignment>) localState.get(Common.LS_LOCAL_ASSIGNMENTS);
            if (localAssignment == null) {
                localAssignment = new HashMap<>();
            }
            localState.put(Common.LS_LOCAL_ASSIGNMENTS, zkAssignment);
        } catch (IOException e) {
            LOG.error("put LS_LOCAL_ASSIGNMENTS " + zkAssignment + " to localState failed");
            throw e;
        }
        /**
         * Step 5: get reloaded topologies
         */
        Set<String> updateTopologies = getUpdateTopologies(localAssignment, zkAssignment, assignments);
        Set<String> reDownloadTopologies = getNeedReDownloadTopologies(localAssignment);
        if (reDownloadTopologies != null) {
            updateTopologies.addAll(reDownloadTopologies);
        }
        /**
         * get upgrade topology ports
         */
        Map<String, Set<Pair<String, Integer>>> upgradeTopologyPorts = getUpgradeTopologies(stormClusterState, localAssignment, zkAssignment);
        if (upgradeTopologyPorts.size() > 0) {
            LOG.info("upgrade topology ports:{}", upgradeTopologyPorts);
            updateTopologies.addAll(upgradeTopologyPorts.keySet());
        }
        /**
         * Step 6: download code from ZK
         */
        Map<String, String> topologyCodes = getTopologyCodeLocations(assignments, supervisorId);
        // downloadFailedTopologyIds which can't finished download binary from nimbus
        Set<String> downloadFailedTopologyIds = new HashSet<>();
        downloadTopology(topologyCodes, downloadedTopologyIds, updateTopologies, assignments, downloadFailedTopologyIds);
        /**
         * Step 7: remove any downloaded useless topology
         */
        removeUselessTopology(topologyCodes, downloadedTopologyIds);
        /**
         * Step 8: push syncProcesses Event
         */
        // processEventManager.add(syncProcesses);
        syncProcesses.run(zkAssignment, downloadFailedTopologyIds, upgradeTopologyPorts);
        // set the trigger to update heartbeat of supervisor
        heartbeat.updateHbTrigger(true);
        try {
            // update localState
            localState.put(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION, assignmentVersion);
            localState.put(Common.LS_LOCAl_ZK_ASSIGNMENTS, assignments);
        } catch (IOException e) {
            LOG.error("put LS_LOCAL_ZK_ASSIGNMENT_VERSION & LS_LOCAl_ZK_ASSIGNMENTS to localState failed");
            throw e;
        }
    } catch (Exception e) {
        LOG.error("Failed to init SyncSupervisorEvent", e);
    // throw new RuntimeException(e);
    }
    if (healthStatus.isMoreSeriousThan(HealthStatus.PANIC)) {
        // if status is panic, kill supervisor;
        JStormUtils.halt_process(0, "Supervisor machine check status: Panic! !!!!shutdown!!!!");
    }
}