Examples with WorkerSlot org.apache.storm.scheduler.WorkerSlot used on opensource projects

Example 36 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class RAS_Nodes method getAllNodesFrom.

public static Map<String, RAS_Node> getAllNodesFrom(Cluster cluster, Topologies topologies) {
    //A map of node ids to node objects
    Map<String, RAS_Node> nodeIdToNode = new HashMap<String, RAS_Node>();
    //A map of assignments organized by node with the following format:
    //{nodeId -> {topologyId -> {workerId -> {execs}}}}
    Map<String, Map<String, Map<String, Collection<ExecutorDetails>>>> assignmentRelationshipMap = new HashMap<String, Map<String, Map<String, Collection<ExecutorDetails>>>>();
    Map<String, Map<String, WorkerSlot>> workerIdToWorker = new HashMap<String, Map<String, WorkerSlot>>();
    for (SchedulerAssignment assignment : cluster.getAssignments().values()) {
        String topId = assignment.getTopologyId();
        for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> entry : assignment.getSlotToExecutors().entrySet()) {
            WorkerSlot slot = entry.getKey();
            String nodeId = slot.getNodeId();
            Collection<ExecutorDetails> execs = entry.getValue();
            if (!assignmentRelationshipMap.containsKey(nodeId)) {
                assignmentRelationshipMap.put(nodeId, new HashMap<String, Map<String, Collection<ExecutorDetails>>>());
                workerIdToWorker.put(nodeId, new HashMap<String, WorkerSlot>());
            }
            workerIdToWorker.get(nodeId).put(slot.getId(), slot);
            if (!assignmentRelationshipMap.get(nodeId).containsKey(topId)) {
                assignmentRelationshipMap.get(nodeId).put(topId, new HashMap<String, Collection<ExecutorDetails>>());
            }
            if (!assignmentRelationshipMap.get(nodeId).get(topId).containsKey(slot.getId())) {
                assignmentRelationshipMap.get(nodeId).get(topId).put(slot.getId(), new LinkedList<ExecutorDetails>());
            }
            assignmentRelationshipMap.get(nodeId).get(topId).get(slot.getId()).addAll(execs);
        }
    }
    for (SupervisorDetails sup : cluster.getSupervisors().values()) {
        //Initialize a worker slot for every port even if there is no assignment to it
        for (int port : sup.getAllPorts()) {
            WorkerSlot worker = new WorkerSlot(sup.getId(), port);
            if (!workerIdToWorker.containsKey(sup.getId())) {
                workerIdToWorker.put(sup.getId(), new HashMap<String, WorkerSlot>());
            }
            if (!workerIdToWorker.get(sup.getId()).containsKey(worker.getId())) {
                workerIdToWorker.get(sup.getId()).put(worker.getId(), worker);
            }
        }
        nodeIdToNode.put(sup.getId(), new RAS_Node(sup.getId(), sup, cluster, topologies, workerIdToWorker.get(sup.getId()), assignmentRelationshipMap.get(sup.getId())));
    }
    //Add in supervisors that might have crashed but workers are still alive
    for (Map.Entry<String, Map<String, Map<String, Collection<ExecutorDetails>>>> entry : assignmentRelationshipMap.entrySet()) {
        String nodeId = entry.getKey();
        Map<String, Map<String, Collection<ExecutorDetails>>> assignments = entry.getValue();
        if (!nodeIdToNode.containsKey(nodeId)) {
            LOG.info("Found an assigned slot(s) on a dead supervisor {} with assignments {}", nodeId, assignments);
            nodeIdToNode.put(nodeId, new RAS_Node(nodeId, null, cluster, topologies, workerIdToWorker.get(nodeId), assignments));
        }
    }
    return nodeIdToNode;
}

Example 37 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class ResourceAwareScheduler method mkAssignment.

private boolean mkAssignment(TopologyDetails td, Map<WorkerSlot, Collection<ExecutorDetails>> schedulerAssignmentMap) {
    if (schedulerAssignmentMap != null) {
        double requestedMemOnHeap = td.getTotalRequestedMemOnHeap();
        double requestedMemOffHeap = td.getTotalRequestedMemOffHeap();
        double requestedCpu = td.getTotalRequestedCpu();
        double assignedMemOnHeap = 0.0;
        double assignedMemOffHeap = 0.0;
        double assignedCpu = 0.0;
        Map<WorkerSlot, Double[]> workerResources = new HashMap<WorkerSlot, Double[]>();
        Set<String> nodesUsed = new HashSet<String>();
        for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> workerToTasksEntry : schedulerAssignmentMap.entrySet()) {
            WorkerSlot targetSlot = workerToTasksEntry.getKey();
            Collection<ExecutorDetails> execsNeedScheduling = workerToTasksEntry.getValue();
            RAS_Node targetNode = this.schedulingState.nodes.getNodeById(targetSlot.getNodeId());
            targetSlot = allocateResourceToSlot(td, execsNeedScheduling, targetSlot);
            targetNode.assign(targetSlot, td, execsNeedScheduling);
            LOG.debug("ASSIGNMENT    TOPOLOGY: {}  TASKS: {} To Node: {} on Slot: {}", td.getName(), execsNeedScheduling, targetNode.getHostname(), targetSlot.getPort());
            for (ExecutorDetails exec : execsNeedScheduling) {
                targetNode.consumeResourcesforTask(exec, td);
            }
            if (!nodesUsed.contains(targetNode.getId())) {
                nodesUsed.add(targetNode.getId());
            }
            assignedMemOnHeap += targetSlot.getAllocatedMemOnHeap();
            assignedMemOffHeap += targetSlot.getAllocatedMemOffHeap();
            assignedCpu += targetSlot.getAllocatedCpu();
            Double[] worker_resources = { requestedMemOnHeap, requestedMemOffHeap, requestedCpu, targetSlot.getAllocatedMemOnHeap(), targetSlot.getAllocatedMemOffHeap(), targetSlot.getAllocatedCpu() };
            workerResources.put(targetSlot, worker_resources);
        }
        Double[] resources = { requestedMemOnHeap, requestedMemOffHeap, requestedCpu, assignedMemOnHeap, assignedMemOffHeap, assignedCpu };
        LOG.debug("setTopologyResources for {}: requested on-heap mem, off-heap mem, cpu: {} {} {} " + "assigned on-heap mem, off-heap mem, cpu: {} {} {}", td.getId(), requestedMemOnHeap, requestedMemOffHeap, requestedCpu, assignedMemOnHeap, assignedMemOffHeap, assignedCpu);
        //updating resources used for a topology
        this.schedulingState.cluster.setTopologyResources(td.getId(), resources);
        this.schedulingState.cluster.setWorkerResources(td.getId(), workerResources);
        return true;
    } else {
        LOG.warn("schedulerAssignmentMap for topo {} is null. This shouldn't happen!", td.getName());
        return false;
    }
}

Example 38 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class ResourceAwareScheduler method allocateResourceToSlot.

private WorkerSlot allocateResourceToSlot(TopologyDetails td, Collection<ExecutorDetails> executors, WorkerSlot slot) {
    double onHeapMem = 0.0;
    double offHeapMem = 0.0;
    double cpu = 0.0;
    for (ExecutorDetails exec : executors) {
        Double onHeapMemForExec = td.getOnHeapMemoryRequirement(exec);
        if (onHeapMemForExec != null) {
            onHeapMem += onHeapMemForExec;
        }
        Double offHeapMemForExec = td.getOffHeapMemoryRequirement(exec);
        if (offHeapMemForExec != null) {
            offHeapMem += offHeapMemForExec;
        }
        Double cpuForExec = td.getTotalCpuReqTask(exec);
        if (cpuForExec != null) {
            cpu += cpuForExec;
        }
    }
    return new WorkerSlot(slot.getNodeId(), slot.getPort(), onHeapMem, offHeapMem, cpu);
}

Example 39 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class StatsUtil method aggWorkerStats.

/**
     * aggregate statistics per worker for a topology. Optionally filtering on specific supervisors
     *
     * @param topologyId       topology id
     * @param topology         storm topology
     * @param task2component   a Map of {task id -> component}, note it's a clojure map
     * @param beats            a converted HashMap of executor heartbeats, {executor -> heartbeat}
     * @param exec2hostPort    a Map of {executor -> host+port}, note it's a clojure map
     * @param includeSys       whether to include system streams
     * @param userAuthorized   whether the user is authorized to view topology info
     * @param filterSupervisor if not null, only return WorkerSummaries for that supervisor
     *
     * @return List<WorkerSummary> thrift structures
     */
public static List<WorkerSummary> aggWorkerStats(String stormId, String stormName, Map<Integer, String> task2Component, Map<List<Integer>, Map<String, Object>> beats, Map<List<Long>, List<Object>> exec2NodePort, Map<String, String> nodeHost, Map<WorkerSlot, WorkerResources> worker2Resources, boolean includeSys, boolean userAuthorized, String filterSupervisor) {
    // host,port => WorkerSummary
    HashMap<WorkerSlot, WorkerSummary> workerSummaryMap = new HashMap<>();
    if (exec2NodePort != null) {
        // for each executor -> node+port pair
        for (Map.Entry<List<Long>, List<Object>> execNodePort : exec2NodePort.entrySet()) {
            List<Object> nodePort = execNodePort.getValue();
            String node = (String) nodePort.get(0);
            Long port = (Long) nodePort.get(1);
            String host = nodeHost.get(node);
            WorkerSlot slot = new WorkerSlot(node, port);
            WorkerResources resources = worker2Resources.get(slot);
            if (filterSupervisor == null || node.equals(filterSupervisor)) {
                WorkerSummary ws = workerSummaryMap.get(slot);
                if (ws == null) {
                    ws = new WorkerSummary();
                    ws.set_host(host);
                    ws.set_port(port.intValue());
                    ws.set_supervisor_id(node);
                    ws.set_topology_id(stormId);
                    ws.set_topology_name(stormName);
                    ws.set_num_executors(0);
                    if (resources != null) {
                        ws.set_assigned_memonheap(resources.get_mem_on_heap());
                        ws.set_assigned_memoffheap(resources.get_mem_off_heap());
                        ws.set_assigned_cpu(resources.get_cpu());
                    } else {
                        ws.set_assigned_memonheap(0);
                        ws.set_assigned_memoffheap(0);
                        ws.set_assigned_cpu(0);
                    }
                    ws.set_component_to_num_tasks(new HashMap<String, Long>());
                    workerSummaryMap.put(slot, ws);
                }
                Map<String, Long> componentToNumTasks = ws.get_component_to_num_tasks();
                // gets min/max task pairs (executors): [1 1] [2 3] ...
                List<Long> exec = execNodePort.getKey();
                // get executor heartbeat
                int hbeatSecs = 0;
                if (beats != null) {
                    Map<String, Object> beat = beats.get(convertExecutor(exec));
                    if (beat != null) {
                        Map<String, Object> hbeat = (Map<String, Object>) beat.get("heartbeat");
                        hbeatSecs = hbeat == null ? 0 : (int) hbeat.get("uptime");
                    }
                }
                ws.set_uptime_secs(hbeatSecs);
                ws.set_num_executors(ws.get_num_executors() + 1);
                // get tasks if the user is authorized for this topology
                if (userAuthorized) {
                    int firstTask = exec.get(0).intValue();
                    int lastTask = exec.get(1).intValue();
                    // get per task components
                    for (int task = firstTask; task <= lastTask; task++) {
                        String component = task2Component.get(task);
                        // them in UI, keep going
                        if (!includeSys && Utils.isSystemId(component)) {
                            continue;
                        }
                        // good to go, increment # of tasks this component is being executed on
                        Long counter = componentToNumTasks.get(component);
                        if (counter == null) {
                            counter = new Long(0);
                        }
                        componentToNumTasks.put(component, counter + 1);
                    }
                }
            }
        }
    }
    return new ArrayList<WorkerSummary>(workerSummaryMap.values());
}

Example 40 with WorkerSlot

use of org.apache.storm.scheduler.WorkerSlot in project storm by apache.

the class Nimbus method computeNewSchedulerAssignments.

private Map<String, SchedulerAssignment> computeNewSchedulerAssignments(Map<String, Assignment> existingAssignments, Topologies topologies, Map<String, StormBase> bases, String scratchTopologyId) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
    Map<String, Set<List<Integer>>> topoToExec = computeTopologyToExecutors(bases);
    updateAllHeartbeats(existingAssignments, topoToExec);
    Map<String, Set<List<Integer>>> topoToAliveExecutors = computeTopologyToAliveExecutors(existingAssignments, topologies, topoToExec, scratchTopologyId);
    Map<String, Set<Long>> supervisorToDeadPorts = computeSupervisorToDeadPorts(existingAssignments, topoToExec, topoToAliveExecutors);
    Map<String, SchedulerAssignmentImpl> topoToSchedAssignment = computeTopologyToSchedulerAssignment(existingAssignments, topoToAliveExecutors);
    Set<String> missingAssignmentTopologies = new HashSet<>();
    for (TopologyDetails topo : topologies.getTopologies()) {
        String id = topo.getId();
        Set<List<Integer>> allExecs = topoToExec.get(id);
        Set<List<Integer>> aliveExecs = topoToAliveExecutors.get(id);
        int numDesiredWorkers = topo.getNumWorkers();
        int numAssignedWorkers = numUsedWorkers(topoToSchedAssignment.get(id));
        if (allExecs == null || allExecs.isEmpty() || !allExecs.equals(aliveExecs) || numDesiredWorkers > numAssignedWorkers) {
            //We have something to schedule...
            missingAssignmentTopologies.add(id);
        }
    }
    Map<String, SupervisorDetails> supervisors = readAllSupervisorDetails(supervisorToDeadPorts, topologies, missingAssignmentTopologies);
    Cluster cluster = new Cluster(inimbus, supervisors, topoToSchedAssignment, conf);
    cluster.setStatusMap(idToSchedStatus.get());
    scheduler.schedule(topologies, cluster);
    //merge with existing statuses
    idToSchedStatus.set(merge(idToSchedStatus.get(), cluster.getStatusMap()));
    nodeIdToResources.set(cluster.getSupervisorsResourcesMap());
    if (!Utils.getBoolean(conf.get(Config.SCHEDULER_DISPLAY_RESOURCE), false)) {
        cluster.updateAssignedMemoryForTopologyAndSupervisor(topologies);
    }
    // This is a hack for non-ras scheduler topology and worker resources
    Map<String, TopologyResources> resources = new HashMap<>();
    for (Entry<String, Double[]> uglyResources : cluster.getTopologyResourcesMap().entrySet()) {
        Double[] r = uglyResources.getValue();
        resources.put(uglyResources.getKey(), new TopologyResources(r[0], r[1], r[2], r[3], r[4], r[5]));
    }
    idToResources.getAndAccumulate(resources, (orig, update) -> merge(orig, update));
    Map<String, Map<WorkerSlot, WorkerResources>> workerResources = new HashMap<>();
    for (Entry<String, Map<WorkerSlot, Double[]>> uglyWorkerResources : cluster.getWorkerResourcesMap().entrySet()) {
        Map<WorkerSlot, WorkerResources> slotToResources = new HashMap<>();
        for (Entry<WorkerSlot, Double[]> uglySlotToResources : uglyWorkerResources.getValue().entrySet()) {
            Double[] r = uglySlotToResources.getValue();
            WorkerResources wr = new WorkerResources();
            wr.set_mem_on_heap(r[0]);
            wr.set_mem_off_heap(r[1]);
            wr.set_cpu(r[2]);
            slotToResources.put(uglySlotToResources.getKey(), wr);
        }
        workerResources.put(uglyWorkerResources.getKey(), slotToResources);
    }
    idToWorkerResources.getAndAccumulate(workerResources, (orig, update) -> merge(orig, update));
    return cluster.getAssignments();
}