Examples with LocalAssignment com.alibaba.jstorm.daemon.worker.LocalAssignment used on opensource projects

Example 1 with LocalAssignment

use of com.alibaba.jstorm.daemon.worker.LocalAssignment in project jstorm by alibaba.

the class SyncSupervisorEvent method getUpdateTopologys.

private Set<String> getUpdateTopologys(Map<Integer, LocalAssignment> localAssignments, Map<Integer, LocalAssignment> zkAssignments, Map<String, Assignment> assignments) {
    Set<String> ret = new HashSet<String>();
    if (localAssignments != null && zkAssignments != null) {
        for (Entry<Integer, LocalAssignment> entry : localAssignments.entrySet()) {
            Integer port = entry.getKey();
            LocalAssignment localAssignment = entry.getValue();
            LocalAssignment zkAssignment = zkAssignments.get(port);
            if (localAssignment == null || zkAssignment == null)
                continue;
            Assignment assignment = assignments.get(localAssignment.getTopologyId());
            if (localAssignment.getTopologyId().equals(zkAssignment.getTopologyId()) && assignment != null && assignment.isTopologyChange(localAssignment.getTimeStamp()))
                if (ret.add(localAssignment.getTopologyId())) {
                    LOG.info("Topology " + localAssignment.getTopologyId() + " has been updated. LocalTs=" + localAssignment.getTimeStamp() + ", ZkTs=" + zkAssignment.getTimeStamp());
                }
        }
    }
    return ret;
}

Example 2 with LocalAssignment

use of com.alibaba.jstorm.daemon.worker.LocalAssignment in project jstorm by alibaba.

the class SyncSupervisorEvent method run.

@Override
public void run() {
    LOG.debug("Synchronizing supervisor, interval seconds:" + TimeUtils.time_delta(lastTime));
    lastTime = TimeUtils.current_time_secs();
    //In order to ensure that the status is the same for each execution of syncsupervisor
    MachineCheckStatus checkStatus = new MachineCheckStatus();
    checkStatus.SetType(heartbeat.getCheckStatus().getType());
    try {
        RunnableCallback syncCallback = new EventManagerZkPusher(this, syncSupEventManager);
        Map<String, Integer> assignmentVersion = (Map<String, Integer>) localState.get(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION);
        if (assignmentVersion == null) {
            assignmentVersion = new HashMap<String, Integer>();
        }
        Map<String, Assignment> assignments = (Map<String, Assignment>) localState.get(Common.LS_LOCAl_ZK_ASSIGNMENTS);
        if (assignments == null) {
            assignments = new HashMap<String, Assignment>();
        }
        LOG.debug("get local assignments  " + assignments);
        LOG.debug("get local assignments version " + assignmentVersion);
        if (checkStatus.getType().equals(MachineCheckStatus.StatusType.panic) || checkStatus.getType().equals(MachineCheckStatus.StatusType.error)) {
            // if statuts is pannic or error, it will clear all assignments and kill all workers;
            assignmentVersion.clear();
            assignments.clear();
            LOG.warn("Supervisor Machine Check Status :" + checkStatus.getType() + ", so kill all workers.");
        } else {
            getAllAssignments(assignmentVersion, assignments, syncCallback);
        }
        LOG.debug("Get all assignments " + assignments);
        /**
             * Step 2: get topologyIds list from STORM-LOCAL-DIR/supervisor/stormdist/
             */
        List<String> downloadedTopologyIds = StormConfig.get_supervisor_toplogy_list(conf);
        LOG.debug("Downloaded storm ids: " + downloadedTopologyIds);
        /**
             * Step 3: get <port,LocalAssignments> from ZK local node's assignment
             */
        Map<Integer, LocalAssignment> zkAssignment;
        zkAssignment = getLocalAssign(stormClusterState, supervisorId, assignments);
        Map<Integer, LocalAssignment> localAssignment;
        /**
             * Step 4: writer local assignment to LocalState
             */
        try {
            LOG.debug("Writing local assignment " + zkAssignment);
            localAssignment = (Map<Integer, LocalAssignment>) localState.get(Common.LS_LOCAL_ASSIGNMENTS);
            if (localAssignment == null) {
                localAssignment = new HashMap<Integer, LocalAssignment>();
            }
            localState.put(Common.LS_LOCAL_ASSIGNMENTS, zkAssignment);
        } catch (IOException e) {
            LOG.error("put LS_LOCAL_ASSIGNMENTS " + zkAssignment + " of localState failed");
            throw e;
        }
        /**
             * Step 5: get reloaded topologys
             */
        Set<String> updateTopologys;
        updateTopologys = getUpdateTopologys(localAssignment, zkAssignment, assignments);
        Set<String> reDownloadTopologys = getNeedReDownloadTopologys(localAssignment);
        if (reDownloadTopologys != null) {
            updateTopologys.addAll(reDownloadTopologys);
        }
        /**
             * Step 6: download code from ZK
             */
        Map<String, String> topologyCodes = getTopologyCodeLocations(assignments, supervisorId);
        // downloadFailedTopologyIds which can't finished download binary from nimbus
        Set<String> downloadFailedTopologyIds = new HashSet<String>();
        downloadTopology(topologyCodes, downloadedTopologyIds, updateTopologys, assignments, downloadFailedTopologyIds);
        /**
             * Step 7: remove any downloaded useless topology
             */
        removeUselessTopology(topologyCodes, downloadedTopologyIds);
        /**
             * Step 7: push syncProcesses Event
             */
        // processEventManager.add(syncProcesses);
        syncProcesses.run(zkAssignment, downloadFailedTopologyIds);
        // If everything is OK, set the trigger to update heartbeat of
        // supervisor
        heartbeat.updateHbTrigger(true);
        try {
            // update localState
            localState.put(Common.LS_LOCAL_ZK_ASSIGNMENT_VERSION, assignmentVersion);
            localState.put(Common.LS_LOCAl_ZK_ASSIGNMENTS, assignments);
        } catch (IOException e) {
            LOG.error("put LS_LOCAL_ZK_ASSIGNMENT_VERSION&&LS_LOCAl_ZK_ASSIGNMENTS  failed");
            throw e;
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
    // throw new RuntimeException(e);
    }
    if (checkStatus.getType().equals(MachineCheckStatus.StatusType.panic)) {
        // if statuts is pannic, it will kill supervisor;
        JStormUtils.halt_process(0, "Supervisor Machine Check Status : Panic , !!!!shutdown!!!!");
    }
}

Example 3 with LocalAssignment

use of com.alibaba.jstorm.daemon.worker.LocalAssignment in project jstorm by alibaba.

the class SyncSupervisorEvent method readMyTasks.

/**
     * get local node's tasks
     * 
     * @param stormClusterState
     * @param topologyId
     * @param supervisorId
     * @return Map: {port, LocalAssignment}
     * @throws Exception
     */
private Map<Integer, LocalAssignment> readMyTasks(StormClusterState stormClusterState, String topologyId, String supervisorId, Assignment assignmentInfo) throws Exception {
    Map<Integer, LocalAssignment> portTasks = new HashMap<Integer, LocalAssignment>();
    Set<ResourceWorkerSlot> workers = assignmentInfo.getWorkers();
    if (workers == null) {
        LOG.error("No worker of assignment's " + assignmentInfo);
        return portTasks;
    }
    for (ResourceWorkerSlot worker : workers) {
        if (!supervisorId.equals(worker.getNodeId()))
            continue;
        portTasks.put(worker.getPort(), new LocalAssignment(topologyId, worker.getTasks(), Common.topologyIdToName(topologyId), worker.getMemSize(), worker.getCpu(), worker.getJvm(), assignmentInfo.getTimeStamp()));
    }
    return portTasks;
}

Example 4 with LocalAssignment

use of com.alibaba.jstorm.daemon.worker.LocalAssignment in project jstorm by alibaba.

the class SyncSupervisorEvent method getLocalAssign.

/**
     * a port must be assigned one topology
     * 
     * @param stormClusterState
     * @param supervisorId
     * @throws Exception
     * @returns map: {port,LocalAssignment}
     */
private Map<Integer, LocalAssignment> getLocalAssign(StormClusterState stormClusterState, String supervisorId, Map<String, Assignment> assignments) throws Exception {
    Map<Integer, LocalAssignment> portLA = new HashMap<Integer, LocalAssignment>();
    for (Entry<String, Assignment> assignEntry : assignments.entrySet()) {
        String topologyId = assignEntry.getKey();
        Assignment assignment = assignEntry.getValue();
        Map<Integer, LocalAssignment> portTasks = readMyTasks(stormClusterState, topologyId, supervisorId, assignment);
        if (portTasks == null) {
            continue;
        }
        // a port must be assigned one storm
        for (Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
            Integer port = entry.getKey();
            LocalAssignment la = entry.getValue();
            if (!portLA.containsKey(port)) {
                portLA.put(port, la);
            } else {
                throw new RuntimeException("Should not have multiple topologys assigned to one port");
            }
        }
    }
    return portLA;
}