Examples with SchedulerAssignment org.apache.storm.scheduler.SchedulerAssignment used on opensource projects

Example 1 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class Nimbus method mkAssignments.

private void mkAssignments(String scratchTopoId) throws Exception {
    if (!isLeader()) {
        LOG.info("not a leader, skipping assignments");
        return;
    }
    // get existing assignment (just the topologyToExecutorToNodePort map) -> default to {}
    // filter out ones which have a executor timeout
    // figure out available slots on cluster. add to that the used valid slots to get total slots. figure out how many executors should be in each slot (e.g., 4, 4, 4, 5)
    // only keep existing slots that satisfy one of those slots. for rest, reassign them across remaining slots
    // edge case for slots with no executor timeout but with supervisor timeout... just treat these as valid slots that can be reassigned to. worst comes to worse the executor will timeout and won't assign here next time around
    IStormClusterState state = stormClusterState;
    //read all the topologies
    Map<String, StormBase> bases;
    Map<String, TopologyDetails> tds = new HashMap<>();
    synchronized (submitLock) {
        bases = state.topologyBases();
        for (Iterator<Entry<String, StormBase>> it = bases.entrySet().iterator(); it.hasNext(); ) {
            Entry<String, StormBase> entry = it.next();
            String id = entry.getKey();
            try {
                tds.put(id, readTopologyDetails(id, entry.getValue()));
            } catch (KeyNotFoundException e) {
                //A race happened and it is probably not running
                it.remove();
            }
        }
    }
    Topologies topologies = new Topologies(tds);
    List<String> assignedTopologyIds = state.assignments(null);
    Map<String, Assignment> existingAssignments = new HashMap<>();
    for (String id : assignedTopologyIds) {
        // will be treated as free slot in the scheduler code.
        if (!id.equals(scratchTopoId)) {
            existingAssignments.put(id, state.assignmentInfo(id, null));
        }
    }
    // make the new assignments for topologies
    Map<String, SchedulerAssignment> newSchedulerAssignments = null;
    synchronized (schedLock) {
        newSchedulerAssignments = computeNewSchedulerAssignments(existingAssignments, topologies, bases, scratchTopoId);
        Map<String, Map<List<Long>, List<Object>>> topologyToExecutorToNodePort = computeNewTopoToExecToNodePort(newSchedulerAssignments, existingAssignments);
        for (String id : assignedTopologyIds) {
            if (!topologyToExecutorToNodePort.containsKey(id)) {
                topologyToExecutorToNodePort.put(id, null);
            }
        }
        Map<String, Map<List<Object>, List<Double>>> newAssignedWorkerToResources = computeTopoToNodePortToResources(newSchedulerAssignments);
        int nowSecs = Time.currentTimeSecs();
        Map<String, SupervisorDetails> basicSupervisorDetailsMap = basicSupervisorDetailsMap(state);
        //construct the final Assignments by adding start-times etc into it
        Map<String, Assignment> newAssignments = new HashMap<>();
        for (Entry<String, Map<List<Long>, List<Object>>> entry : topologyToExecutorToNodePort.entrySet()) {
            String topoId = entry.getKey();
            Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            Set<String> allNodes = new HashSet<>();
            if (execToNodePort != null) {
                for (List<Object> nodePort : execToNodePort.values()) {
                    allNodes.add((String) nodePort.get(0));
                }
            }
            Map<String, String> allNodeHost = new HashMap<>();
            if (existingAssignment != null) {
                allNodeHost.putAll(existingAssignment.get_node_host());
            }
            for (String node : allNodes) {
                String host = inimbus.getHostName(basicSupervisorDetailsMap, node);
                if (host != null) {
                    allNodeHost.put(node, host);
                }
            }
            Map<List<Long>, NodeInfo> execNodeInfo = null;
            if (existingAssignment != null) {
                execNodeInfo = existingAssignment.get_executor_node_port();
            }
            List<List<Long>> reassignExecutors = changedExecutors(execNodeInfo, execToNodePort);
            Map<List<Long>, Long> startTimes = new HashMap<>();
            if (existingAssignment != null) {
                startTimes.putAll(existingAssignment.get_executor_start_time_secs());
            }
            for (List<Long> id : reassignExecutors) {
                startTimes.put(id, (long) nowSecs);
            }
            Map<List<Object>, List<Double>> workerToResources = newAssignedWorkerToResources.get(topoId);
            Assignment newAssignment = new Assignment((String) conf.get(Config.STORM_LOCAL_DIR));
            Map<String, String> justAssignedKeys = new HashMap<>(allNodeHost);
            //Modifies justAssignedKeys
            justAssignedKeys.keySet().retainAll(allNodes);
            newAssignment.set_node_host(justAssignedKeys);
            //convert NodePort to NodeInfo (again!!!).
            Map<List<Long>, NodeInfo> execToNodeInfo = new HashMap<>();
            for (Entry<List<Long>, List<Object>> execAndNodePort : execToNodePort.entrySet()) {
                List<Object> nodePort = execAndNodePort.getValue();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                execToNodeInfo.put(execAndNodePort.getKey(), ni);
            }
            newAssignment.set_executor_node_port(execToNodeInfo);
            newAssignment.set_executor_start_time_secs(startTimes);
            //do another conversion (lets just make this all common)
            Map<NodeInfo, WorkerResources> workerResources = new HashMap<>();
            for (Entry<List<Object>, List<Double>> wr : workerToResources.entrySet()) {
                List<Object> nodePort = wr.getKey();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                List<Double> r = wr.getValue();
                WorkerResources resources = new WorkerResources();
                resources.set_mem_on_heap(r.get(0));
                resources.set_mem_off_heap(r.get(1));
                resources.set_cpu(r.get(2));
                workerResources.put(ni, resources);
            }
            newAssignment.set_worker_resources(workerResources);
            newAssignments.put(topoId, newAssignment);
        }
        if (!newAssignments.equals(existingAssignments)) {
            LOG.debug("RESETTING id->resources and id->worker-resources cache!");
            idToResources.set(new HashMap<>());
            idToWorkerResources.set(new HashMap<>());
        }
        // only log/set when there's been a change to the assignment
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            //NOT Used TopologyDetails topologyDetails = topologies.getById(topoId);
            if (assignment.equals(existingAssignment)) {
                LOG.debug("Assignment for {} hasn't changed", topoId);
            } else {
                LOG.info("Setting new assignment for topology id {}: {}", topoId, assignment);
                state.setAssignment(topoId, assignment);
            }
        }
        Map<String, Collection<WorkerSlot>> addedSlots = new HashMap<>();
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            if (existingAssignment == null) {
                existingAssignment = new Assignment();
                existingAssignment.set_executor_node_port(new HashMap<>());
                existingAssignment.set_executor_start_time_secs(new HashMap<>());
            }
            Set<WorkerSlot> newSlots = newlyAddedSlots(existingAssignment, assignment);
            addedSlots.put(topoId, newSlots);
        }
        inimbus.assignSlots(topologies, addedSlots);
    }
}

Example 2 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class Nimbus method computeTopologyToSchedulerAssignment.

/**
     * Convert assignment information in zk to SchedulerAssignment, so it can be used by scheduler api.
     * @param existingAssignments current assignments
     * @param topologyToAliveExecutors executors that are alive
     * @return topo ID to schedulerAssignment
     */
private Map<String, SchedulerAssignmentImpl> computeTopologyToSchedulerAssignment(Map<String, Assignment> existingAssignments, Map<String, Set<List<Integer>>> topologyToAliveExecutors) {
    Map<String, SchedulerAssignmentImpl> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignments.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        Set<List<Integer>> aliveExecutors = topologyToAliveExecutors.get(topoId);
        Map<List<Long>, NodeInfo> execToNodePort = assignment.get_executor_node_port();
        Map<NodeInfo, WorkerResources> workerToResources = assignment.get_worker_resources();
        Map<NodeInfo, WorkerSlot> nodePortToSlot = new HashMap<>();
        for (Entry<NodeInfo, WorkerResources> nodeAndResources : workerToResources.entrySet()) {
            NodeInfo info = nodeAndResources.getKey();
            WorkerResources resources = nodeAndResources.getValue();
            WorkerSlot slot = new WorkerSlot(info.get_node(), info.get_port_iterator().next(), resources.get_mem_on_heap(), resources.get_mem_off_heap(), resources.get_cpu());
            nodePortToSlot.put(info, slot);
        }
        Map<ExecutorDetails, WorkerSlot> execToSlot = new HashMap<>();
        for (Entry<List<Long>, NodeInfo> execAndNodePort : execToNodePort.entrySet()) {
            List<Integer> exec = asIntExec(execAndNodePort.getKey());
            NodeInfo info = execAndNodePort.getValue();
            if (aliveExecutors.contains(exec)) {
                execToSlot.put(new ExecutorDetails(exec.get(0), exec.get(1)), nodePortToSlot.get(info));
            }
        }
        ret.put(topoId, new SchedulerAssignmentImpl(topoId, execToSlot));
    }
    return ret;
}

Example 3 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class Nimbus method computeTopoToNodePortToResources.

/**
     * convert {topology-id -> SchedulerAssignment} to
     *         {topology-id -> {[node port] [mem-on-heap mem-off-heap cpu]}}
     * Make sure this can deal with other non-RAS schedulers
     * later we may further support map-for-any-resources
     * @param schedAssignments the assignments
     * @return  {topology-id {[node port] [mem-on-heap mem-off-heap cpu]}}
     */
private static Map<String, Map<List<Object>, List<Double>>> computeTopoToNodePortToResources(Map<String, SchedulerAssignment> schedAssignments) {
    Map<String, Map<List<Object>, List<Double>>> ret = new HashMap<>();
    for (Entry<String, SchedulerAssignment> schedEntry : schedAssignments.entrySet()) {
        Map<List<Object>, List<Double>> nodePortToResources = new HashMap<>();
        for (WorkerSlot slot : schedEntry.getValue().getExecutorToSlot().values()) {
            List<Object> nodePort = new ArrayList<>(2);
            nodePort.add(slot.getNodeId());
            nodePort.add((long) slot.getPort());
            List<Double> resources = new ArrayList<>(3);
            resources.add(slot.getAllocatedMemOnHeap());
            resources.add(slot.getAllocatedMemOffHeap());
            resources.add(slot.getAllocatedCpu());
            nodePortToResources.put(nodePort, resources);
        }
        ret.put(schedEntry.getKey(), nodePortToResources);
    }
    return ret;
}

Example 4 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class Nimbus method computeTopoToExecToNodePort.

/**
     * convert {topology-id -> SchedulerAssignment} to
     *         {topology-id -> {executor [node port]}}
     * @return
     */
private static Map<String, Map<List<Long>, List<Object>>> computeTopoToExecToNodePort(Map<String, SchedulerAssignment> schedAssignments) {
    Map<String, Map<List<Long>, List<Object>>> ret = new HashMap<>();
    for (Entry<String, SchedulerAssignment> schedEntry : schedAssignments.entrySet()) {
        Map<List<Long>, List<Object>> execToNodePort = new HashMap<>();
        for (Entry<ExecutorDetails, WorkerSlot> execAndNodePort : schedEntry.getValue().getExecutorToSlot().entrySet()) {
            ExecutorDetails exec = execAndNodePort.getKey();
            WorkerSlot slot = execAndNodePort.getValue();
            List<Long> listExec = new ArrayList<>(2);
            listExec.add((long) exec.getStartTask());
            listExec.add((long) exec.getEndTask());
            List<Object> nodePort = new ArrayList<>(2);
            nodePort.add(slot.getNodeId());
            nodePort.add((long) slot.getPort());
            execToNodePort.put(listExec, nodePort);
        }
        ret.put(schedEntry.getKey(), execToNodePort);
    }
    return ret;
}

Example 5 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class IsolationScheduler method hostAssignments.

private Map<String, List<AssignmentInfo>> hostAssignments(Cluster cluster) {
    Collection<SchedulerAssignment> assignmentValues = cluster.getAssignments().values();
    Map<String, List<AssignmentInfo>> hostAssignments = new HashMap<String, List<AssignmentInfo>>();
    for (SchedulerAssignment sa : assignmentValues) {
        Map<WorkerSlot, List<ExecutorDetails>> slotExecutors = Utils.reverseMap(sa.getExecutorToSlot());
        Set<Map.Entry<WorkerSlot, List<ExecutorDetails>>> entries = slotExecutors.entrySet();
        for (Map.Entry<WorkerSlot, List<ExecutorDetails>> entry : entries) {
            WorkerSlot slot = entry.getKey();
            List<ExecutorDetails> executors = entry.getValue();
            String host = cluster.getHost(slot.getNodeId());
            AssignmentInfo ass = new AssignmentInfo(slot, sa.getTopologyId(), new HashSet<ExecutorDetails>(executors));
            List<AssignmentInfo> executorList = hostAssignments.get(host);
            if (executorList == null) {
                executorList = new ArrayList<AssignmentInfo>();
                hostAssignments.put(host, executorList);
            }
            executorList.add(ass);
        }
    }
    return hostAssignments;
}