Examples with TopologyDetails org.apache.storm.scheduler.TopologyDetails used on opensource projects

Example 1 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class Nimbus method mkAssignments.

private void mkAssignments(String scratchTopoId) throws Exception {
    if (!isLeader()) {
        LOG.info("not a leader, skipping assignments");
        return;
    }
    // get existing assignment (just the topologyToExecutorToNodePort map) -> default to {}
    // filter out ones which have a executor timeout
    // figure out available slots on cluster. add to that the used valid slots to get total slots. figure out how many executors should be in each slot (e.g., 4, 4, 4, 5)
    // only keep existing slots that satisfy one of those slots. for rest, reassign them across remaining slots
    // edge case for slots with no executor timeout but with supervisor timeout... just treat these as valid slots that can be reassigned to. worst comes to worse the executor will timeout and won't assign here next time around
    IStormClusterState state = stormClusterState;
    //read all the topologies
    Map<String, StormBase> bases;
    Map<String, TopologyDetails> tds = new HashMap<>();
    synchronized (submitLock) {
        bases = state.topologyBases();
        for (Iterator<Entry<String, StormBase>> it = bases.entrySet().iterator(); it.hasNext(); ) {
            Entry<String, StormBase> entry = it.next();
            String id = entry.getKey();
            try {
                tds.put(id, readTopologyDetails(id, entry.getValue()));
            } catch (KeyNotFoundException e) {
                //A race happened and it is probably not running
                it.remove();
            }
        }
    }
    Topologies topologies = new Topologies(tds);
    List<String> assignedTopologyIds = state.assignments(null);
    Map<String, Assignment> existingAssignments = new HashMap<>();
    for (String id : assignedTopologyIds) {
        // will be treated as free slot in the scheduler code.
        if (!id.equals(scratchTopoId)) {
            existingAssignments.put(id, state.assignmentInfo(id, null));
        }
    }
    // make the new assignments for topologies
    Map<String, SchedulerAssignment> newSchedulerAssignments = null;
    synchronized (schedLock) {
        newSchedulerAssignments = computeNewSchedulerAssignments(existingAssignments, topologies, bases, scratchTopoId);
        Map<String, Map<List<Long>, List<Object>>> topologyToExecutorToNodePort = computeNewTopoToExecToNodePort(newSchedulerAssignments, existingAssignments);
        for (String id : assignedTopologyIds) {
            if (!topologyToExecutorToNodePort.containsKey(id)) {
                topologyToExecutorToNodePort.put(id, null);
            }
        }
        Map<String, Map<List<Object>, List<Double>>> newAssignedWorkerToResources = computeTopoToNodePortToResources(newSchedulerAssignments);
        int nowSecs = Time.currentTimeSecs();
        Map<String, SupervisorDetails> basicSupervisorDetailsMap = basicSupervisorDetailsMap(state);
        //construct the final Assignments by adding start-times etc into it
        Map<String, Assignment> newAssignments = new HashMap<>();
        for (Entry<String, Map<List<Long>, List<Object>>> entry : topologyToExecutorToNodePort.entrySet()) {
            String topoId = entry.getKey();
            Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            Set<String> allNodes = new HashSet<>();
            if (execToNodePort != null) {
                for (List<Object> nodePort : execToNodePort.values()) {
                    allNodes.add((String) nodePort.get(0));
                }
            }
            Map<String, String> allNodeHost = new HashMap<>();
            if (existingAssignment != null) {
                allNodeHost.putAll(existingAssignment.get_node_host());
            }
            for (String node : allNodes) {
                String host = inimbus.getHostName(basicSupervisorDetailsMap, node);
                if (host != null) {
                    allNodeHost.put(node, host);
                }
            }
            Map<List<Long>, NodeInfo> execNodeInfo = null;
            if (existingAssignment != null) {
                execNodeInfo = existingAssignment.get_executor_node_port();
            }
            List<List<Long>> reassignExecutors = changedExecutors(execNodeInfo, execToNodePort);
            Map<List<Long>, Long> startTimes = new HashMap<>();
            if (existingAssignment != null) {
                startTimes.putAll(existingAssignment.get_executor_start_time_secs());
            }
            for (List<Long> id : reassignExecutors) {
                startTimes.put(id, (long) nowSecs);
            }
            Map<List<Object>, List<Double>> workerToResources = newAssignedWorkerToResources.get(topoId);
            Assignment newAssignment = new Assignment((String) conf.get(Config.STORM_LOCAL_DIR));
            Map<String, String> justAssignedKeys = new HashMap<>(allNodeHost);
            //Modifies justAssignedKeys
            justAssignedKeys.keySet().retainAll(allNodes);
            newAssignment.set_node_host(justAssignedKeys);
            //convert NodePort to NodeInfo (again!!!).
            Map<List<Long>, NodeInfo> execToNodeInfo = new HashMap<>();
            for (Entry<List<Long>, List<Object>> execAndNodePort : execToNodePort.entrySet()) {
                List<Object> nodePort = execAndNodePort.getValue();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                execToNodeInfo.put(execAndNodePort.getKey(), ni);
            }
            newAssignment.set_executor_node_port(execToNodeInfo);
            newAssignment.set_executor_start_time_secs(startTimes);
            //do another conversion (lets just make this all common)
            Map<NodeInfo, WorkerResources> workerResources = new HashMap<>();
            for (Entry<List<Object>, List<Double>> wr : workerToResources.entrySet()) {
                List<Object> nodePort = wr.getKey();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                List<Double> r = wr.getValue();
                WorkerResources resources = new WorkerResources();
                resources.set_mem_on_heap(r.get(0));
                resources.set_mem_off_heap(r.get(1));
                resources.set_cpu(r.get(2));
                workerResources.put(ni, resources);
            }
            newAssignment.set_worker_resources(workerResources);
            newAssignments.put(topoId, newAssignment);
        }
        if (!newAssignments.equals(existingAssignments)) {
            LOG.debug("RESETTING id->resources and id->worker-resources cache!");
            idToResources.set(new HashMap<>());
            idToWorkerResources.set(new HashMap<>());
        }
        // only log/set when there's been a change to the assignment
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            //NOT Used TopologyDetails topologyDetails = topologies.getById(topoId);
            if (assignment.equals(existingAssignment)) {
                LOG.debug("Assignment for {} hasn't changed", topoId);
            } else {
                LOG.info("Setting new assignment for topology id {}: {}", topoId, assignment);
                state.setAssignment(topoId, assignment);
            }
        }
        Map<String, Collection<WorkerSlot>> addedSlots = new HashMap<>();
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            if (existingAssignment == null) {
                existingAssignment = new Assignment();
                existingAssignment.set_executor_node_port(new HashMap<>());
                existingAssignment.set_executor_start_time_secs(new HashMap<>());
            }
            Set<WorkerSlot> newSlots = newlyAddedSlots(existingAssignment, assignment);
            addedSlots.put(topoId, newSlots);
        }
        inimbus.assignSlots(topologies, addedSlots);
    }
}

Example 2 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class Nimbus method readTopologyDetails.

private TopologyDetails readTopologyDetails(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
    assert (base != null);
    assert (topoId != null);
    BlobStore store = blobStore;
    Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
    StormTopology topo = readStormTopologyAsNimbus(topoId, store);
    Map<List<Integer>, String> rawExecToComponent = computeExecutorToComponent(topoId, base);
    Map<ExecutorDetails, String> executorsToComponent = new HashMap<>();
    for (Entry<List<Integer>, String> entry : rawExecToComponent.entrySet()) {
        List<Integer> execs = entry.getKey();
        ExecutorDetails execDetails = new ExecutorDetails(execs.get(0), execs.get(1));
        executorsToComponent.put(execDetails, entry.getValue());
    }
    return new TopologyDetails(topoId, topoConf, topo, base.get_num_workers(), executorsToComponent, base.get_launch_time_secs());
}

Example 3 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class IsolationScheduler method isolatedTopologies.

private List<TopologyDetails> isolatedTopologies(Collection<TopologyDetails> topologies) {
    Set<String> topologyNames = isoMachines.keySet();
    List<TopologyDetails> isoTopologies = new ArrayList<TopologyDetails>();
    for (TopologyDetails topo : topologies) {
        if (topologyNames.contains(topo.getName())) {
            isoTopologies.add(topo);
        }
    }
    return isoTopologies;
}

Example 4 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class DefaultPool method scheduleAsNeeded.

@Override
public void scheduleAsNeeded(NodePool... lesserPools) {
    for (TopologyDetails td : _tds.values()) {
        String topId = td.getId();
        if (_cluster.needsScheduling(td)) {
            LOG.debug("Scheduling topology {}", topId);
            int totalTasks = td.getExecutors().size();
            int origRequest = td.getNumWorkers();
            int slotsRequested = Math.min(totalTasks, origRequest);
            int slotsUsed = Node.countSlotsUsed(topId, _nodes);
            int slotsFree = Node.countFreeSlotsAlive(_nodes);
            //Check to see if we have enough slots before trying to get them
            int slotsAvailable = 0;
            if (slotsRequested > slotsFree) {
                slotsAvailable = NodePool.slotsAvailable(lesserPools);
            }
            int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree + slotsAvailable);
            int executorsNotRunning = _cluster.getUnassignedExecutors(td).size();
            LOG.debug("Slots... requested {} used {} free {} available {} to be used {}, executors not running {}", slotsRequested, slotsUsed, slotsFree, slotsAvailable, slotsToUse, executorsNotRunning);
            if (slotsToUse <= 0) {
                if (executorsNotRunning > 0) {
                    _cluster.setStatus(topId, "Not fully scheduled (No free slots in default pool) " + executorsNotRunning + " executors not scheduled");
                } else {
                    if (slotsUsed < slotsRequested) {
                        _cluster.setStatus(topId, "Running with fewer slots than requested (" + slotsUsed + "/" + origRequest + ")");
                    } else {
                        //slotsUsed < origRequest
                        _cluster.setStatus(topId, "Fully Scheduled (requested " + origRequest + " slots, but could only use " + slotsUsed + ")");
                    }
                }
                continue;
            }
            int slotsNeeded = slotsToUse - slotsFree;
            if (slotsNeeded > 0) {
                _nodes.addAll(NodePool.takeNodesBySlot(slotsNeeded, lesserPools));
            }
            if (executorsNotRunning <= 0) {
                //There are free slots that we can take advantage of now.
                for (Node n : _nodes) {
                    n.freeTopology(topId, _cluster);
                }
                slotsFree = Node.countFreeSlotsAlive(_nodes);
                slotsToUse = Math.min(slotsRequested, slotsFree);
            }
            RoundRobinSlotScheduler slotSched = new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
            LinkedList<Node> nodes = new LinkedList<>(_nodes);
            while (true) {
                Node n;
                do {
                    if (nodes.isEmpty()) {
                        throw new IllegalStateException("This should not happen, we" + " messed up and did not get enough slots");
                    }
                    n = nodes.peekFirst();
                    if (n.totalSlotsFree() == 0) {
                        nodes.remove();
                        n = null;
                    }
                } while (n == null);
                if (!slotSched.assignSlotTo(n)) {
                    break;
                }
            }
            int afterSchedSlotsUsed = Node.countSlotsUsed(topId, _nodes);
            if (afterSchedSlotsUsed < slotsRequested) {
                _cluster.setStatus(topId, "Running with fewer slots than requested (" + afterSchedSlotsUsed + "/" + origRequest + ")");
            } else if (afterSchedSlotsUsed < origRequest) {
                _cluster.setStatus(topId, "Fully Scheduled (requested " + origRequest + " slots, but could only use " + afterSchedSlotsUsed + ")");
            } else {
                _cluster.setStatus(topId, "Fully Scheduled");
            }
        } else {
            _cluster.setStatus(topId, "Fully Scheduled");
        }
    }
}

Example 5 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class IsolatedPool method scheduleAsNeeded.

@Override
public void scheduleAsNeeded(NodePool... lesserPools) {
    for (String topId : _topologyIdToNodes.keySet()) {
        TopologyDetails td = _tds.get(topId);
        Set<Node> allNodes = _topologyIdToNodes.get(topId);
        Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES);
        Integer effectiveNodesRequested = null;
        if (nodesRequested != null) {
            effectiveNodesRequested = Math.min(td.getExecutors().size(), +nodesRequested.intValue());
        }
        if (_cluster.needsScheduling(td) || (effectiveNodesRequested != null && allNodes.size() != effectiveNodesRequested)) {
            LOG.debug("Scheduling topology {}", topId);
            int slotsToUse = 0;
            if (effectiveNodesRequested == null) {
                slotsToUse = getNodesForNotIsolatedTop(td, allNodes, lesserPools);
            } else {
                slotsToUse = getNodesForIsolatedTop(td, allNodes, lesserPools, effectiveNodesRequested);
            }
            //No slots to schedule for some reason, so skip it.
            if (slotsToUse <= 0) {
                continue;
            }
            RoundRobinSlotScheduler slotSched = new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
            LOG.debug("Nodes sorted by free space {}", allNodes);
            while (true) {
                Node n = findBestNode(allNodes);
                if (n == null) {
                    LOG.error("No nodes to use to assign topology {}", td.getName());
                    break;
                }
                if (!slotSched.assignSlotTo(n)) {
                    break;
                }
            }
        }
        Set<Node> found = _topologyIdToNodes.get(topId);
        int nc = found == null ? 0 : found.size();
        _cluster.setStatus(topId, "Scheduled Isolated on " + nc + " Nodes");
    }
}