Examples with TopologyDetails backtype.storm.scheduler.TopologyDetails used on opensource projects

Example 1 with TopologyDetails

use of backtype.storm.scheduler.TopologyDetails in project storm-mesos by nathanmarz.

the class MesosNimbus method allSlotsAvailableForScheduling.

@Override
public Collection<WorkerSlot> allSlotsAvailableForScheduling(Collection<SupervisorDetails> existingSupervisors, Topologies topologies, Set<String> topologiesMissingAssignments) {
    synchronized (OFFERS_LOCK) {
        LOG.info("Currently have " + _offers.size() + " offers buffered");
        if (!topologiesMissingAssignments.isEmpty()) {
            LOG.info("Topologies that need assignments: " + topologiesMissingAssignments.toString());
        }
    }
    Integer cpu = null;
    Integer mem = null;
    // it will mess up scheduling on this cluster permanently 
    for (String id : topologiesMissingAssignments) {
        TopologyDetails details = topologies.getById(id);
        int tcpu = MesosCommon.topologyCpu(_conf, details);
        int tmem = MesosCommon.topologyMem(_conf, details);
        if (cpu == null || tcpu > cpu) {
            cpu = tcpu;
        }
        if (mem == null || tmem > mem) {
            mem = tmem;
        }
    }
    // need access to how many slots are currently used to limit number of slots taken up
    List<WorkerSlot> allSlots = new ArrayList();
    if (cpu != null && mem != null) {
        synchronized (OFFERS_LOCK) {
            for (Offer offer : _offers.values()) {
                allSlots.addAll(toSlots(offer, cpu, mem));
            }
        }
    }
    LOG.info("Number of available slots: " + allSlots.size());
    return allSlots;
}

Example 2 with TopologyDetails

use of backtype.storm.scheduler.TopologyDetails in project jstorm by alibaba.

the class IsolatedPool method scheduleAsNeeded.

@Override
public void scheduleAsNeeded(NodePool... lesserPools) {
    for (String topId : _topologyIdToNodes.keySet()) {
        TopologyDetails td = _tds.get(topId);
        if (_cluster.needsScheduling(td)) {
            LOG.debug("Scheduling topology {}", topId);
            Set<Node> allNodes = _topologyIdToNodes.get(topId);
            Number nodesRequested = (Number) td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES);
            int slotsToUse = 0;
            if (nodesRequested == null) {
                slotsToUse = getNodesForNotIsolatedTop(td, allNodes, lesserPools);
            } else {
                slotsToUse = getNodesForIsolatedTop(td, allNodes, lesserPools, nodesRequested.intValue());
            }
            // No slots to schedule for some reason, so skip it.
            if (slotsToUse <= 0) {
                continue;
            }
            RoundRobinSlotScheduler slotSched = new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
            LinkedList<Node> sortedNodes = new LinkedList<Node>(allNodes);
            Collections.sort(sortedNodes, Node.FREE_NODE_COMPARATOR_DEC);
            LOG.debug("Nodes sorted by free space {}", sortedNodes);
            while (true) {
                Node n = sortedNodes.remove();
                if (!slotSched.assignSlotTo(n)) {
                    break;
                }
                int freeSlots = n.totalSlotsFree();
                for (int i = 0; i < sortedNodes.size(); i++) {
                    if (freeSlots >= sortedNodes.get(i).totalSlotsFree()) {
                        sortedNodes.add(i, n);
                        n = null;
                        break;
                    }
                }
                if (n != null) {
                    sortedNodes.add(n);
                }
            }
        }
        Set<Node> found = _topologyIdToNodes.get(topId);
        int nc = found == null ? 0 : found.size();
        _cluster.setStatus(topId, "Scheduled Isolated on " + nc + " Nodes");
    }
}

Example 3 with TopologyDetails

use of backtype.storm.scheduler.TopologyDetails in project jstorm by alibaba.

the class MultitenantScheduler method schedule.

@Override
public void schedule(Topologies topologies, Cluster cluster) {
    LOG.debug("Rerunning scheduling...");
    Map<String, Node> nodeIdToNode = Node.getAllNodesFrom(cluster);
    Map<String, Number> userConf = getUserConf();
    Map<String, IsolatedPool> userPools = new HashMap<String, IsolatedPool>();
    for (Map.Entry<String, Number> entry : userConf.entrySet()) {
        userPools.put(entry.getKey(), new IsolatedPool(entry.getValue().intValue()));
    }
    DefaultPool defaultPool = new DefaultPool();
    FreePool freePool = new FreePool();
    freePool.init(cluster, nodeIdToNode);
    for (IsolatedPool pool : userPools.values()) {
        pool.init(cluster, nodeIdToNode);
    }
    defaultPool.init(cluster, nodeIdToNode);
    for (TopologyDetails td : topologies.getTopologies()) {
        String user = (String) td.getConf().get(Config.TOPOLOGY_SUBMITTER_USER);
        LOG.debug("Found top {} run by user {}", td.getId(), user);
        NodePool pool = userPools.get(user);
        if (pool == null || !pool.canAdd(td)) {
            pool = defaultPool;
        }
        pool.addTopology(td);
    }
    // Now schedule all of the topologies that need to be scheduled
    for (IsolatedPool pool : userPools.values()) {
        pool.scheduleAsNeeded(freePool, defaultPool);
    }
    defaultPool.scheduleAsNeeded(freePool);
    LOG.debug("Scheduling done...");
}

Example 4 with TopologyDetails

use of backtype.storm.scheduler.TopologyDetails in project storm-lib by xumingming.

the class DemoScheduler method schedule.

public void schedule(Topologies topologies, Cluster cluster) {
    System.out.println("DemoScheduler: begin scheduling");
    // Gets the topology which we want to schedule
    TopologyDetails topology = topologies.getByName("special-topology");
    // make sure the special topology is submitted,
    if (topology != null) {
        boolean needsScheduling = cluster.needsScheduling(topology);
        if (!needsScheduling) {
            System.out.println("Our special topology DOES NOT NEED scheduling.");
        } else {
            System.out.println("Our special topology needs scheduling.");
            // find out all the needs-scheduling components of this topology
            Map<String, List<ExecutorDetails>> componentToExecutors = cluster.getNeedsSchedulingComponentToExecutors(topology);
            System.out.println("needs scheduling(component->executor): " + componentToExecutors);
            System.out.println("needs scheduling(executor->compoenents): " + cluster.getNeedsSchedulingExecutorToComponents(topology));
            SchedulerAssignment currentAssignment = cluster.getAssignmentById(topologies.getByName("special-topology").getId());
            if (currentAssignment != null) {
                System.out.println("current assignments: " + currentAssignment.getExecutorToSlot());
            } else {
                System.out.println("current assignments: {}");
            }
            if (!componentToExecutors.containsKey("special-spout")) {
                System.out.println("Our special-spout DOES NOT NEED scheduling.");
            } else {
                System.out.println("Our special-spout needs scheduling.");
                List<ExecutorDetails> executors = componentToExecutors.get("special-spout");
                // find out the our "special-supervisor" from the supervisor metadata
                Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
                SupervisorDetails specialSupervisor = null;
                for (SupervisorDetails supervisor : supervisors) {
                    Map meta = (Map) supervisor.getSchedulerMeta();
                    if (meta.get("name").equals("special-supervisor")) {
                        specialSupervisor = supervisor;
                        break;
                    }
                }
                // found the special supervisor
                if (specialSupervisor != null) {
                    System.out.println("Found the special-supervisor");
                    List<WorkerSlot> availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // TODO for simplicity, we free all the used slots on the supervisor.
                    if (availableSlots.isEmpty() && !executors.isEmpty()) {
                        for (Integer port : cluster.getUsedPorts(specialSupervisor)) {
                            cluster.freeSlot(new WorkerSlot(specialSupervisor.getId(), port));
                        }
                    }
                    // re-get the aviableSlots
                    availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // since it is just a demo, to keep things simple, we assign all the
                    // executors into one slot.
                    cluster.assign(availableSlots.get(0), topology.getId(), executors);
                    System.out.println("We assigned executors:" + executors + " to slot: [" + availableSlots.get(0).getNodeId() + ", " + availableSlots.get(0).getPort() + "]");
                } else {
                    System.out.println("There is no supervisor named special-supervisor!!!");
                }
            }
        }
    }
    // let system's even scheduler handle the rest scheduling work
    // you can also use your own other scheduler here, this is what
    // makes storm's scheduler composable.
    new EvenScheduler().schedule(topologies, cluster);
}

Example 5 with TopologyDetails

use of backtype.storm.scheduler.TopologyDetails in project jstorm by alibaba.

the class DefaultPool method scheduleAsNeeded.

@Override
public void scheduleAsNeeded(NodePool... lesserPools) {
    for (TopologyDetails td : _tds.values()) {
        String topId = td.getId();
        if (_cluster.needsScheduling(td)) {
            LOG.debug("Scheduling topology {}", topId);
            int totalTasks = td.getExecutors().size();
            int origRequest = td.getNumWorkers();
            int slotsRequested = Math.min(totalTasks, origRequest);
            int slotsUsed = Node.countSlotsUsed(topId, _nodes);
            int slotsFree = Node.countFreeSlotsAlive(_nodes);
            // Check to see if we have enough slots before trying to get them
            int slotsAvailable = 0;
            if (slotsRequested > slotsFree) {
                slotsAvailable = NodePool.slotsAvailable(lesserPools);
            }
            int slotsToUse = Math.min(slotsRequested - slotsUsed, slotsFree + slotsAvailable);
            int executorsNotRunning = _cluster.getUnassignedExecutors(td).size();
            LOG.debug("Slots... requested {} used {} free {} available {} to be used {}, executors not running {}", new Object[] { slotsRequested, slotsUsed, slotsFree, slotsAvailable, slotsToUse, executorsNotRunning });
            if (slotsToUse <= 0) {
                if (executorsNotRunning > 0) {
                    _cluster.setStatus(topId, "Not fully scheduled (No free slots in default pool) " + executorsNotRunning + " executors not scheduled");
                } else {
                    if (slotsUsed < slotsRequested) {
                        _cluster.setStatus(topId, "Running with fewer slots than requested (" + slotsUsed + "/" + origRequest + ")");
                    } else {
                        // slotsUsed < origRequest
                        _cluster.setStatus(topId, "Fully Scheduled (requested " + origRequest + " slots, but could only use " + slotsUsed + ")");
                    }
                }
                continue;
            }
            int slotsNeeded = slotsToUse - slotsFree;
            if (slotsNeeded > 0) {
                _nodes.addAll(NodePool.takeNodesBySlot(slotsNeeded, lesserPools));
            }
            if (executorsNotRunning <= 0) {
                // There are free slots that we can take advantage of now.
                for (Node n : _nodes) {
                    n.freeTopology(topId, _cluster);
                }
                slotsFree = Node.countFreeSlotsAlive(_nodes);
                slotsToUse = Math.min(slotsRequested, slotsFree);
            }
            RoundRobinSlotScheduler slotSched = new RoundRobinSlotScheduler(td, slotsToUse, _cluster);
            LinkedList<Node> nodes = new LinkedList<Node>(_nodes);
            while (true) {
                Node n = null;
                do {
                    if (nodes.isEmpty()) {
                        throw new IllegalStateException("This should not happen, we" + " messed up and did not get enough slots");
                    }
                    n = nodes.peekFirst();
                    if (n.totalSlotsFree() == 0) {
                        nodes.remove();
                        n = null;
                    }
                } while (n == null);
                if (!slotSched.assignSlotTo(n)) {
                    break;
                }
            }
            int afterSchedSlotsUsed = Node.countSlotsUsed(topId, _nodes);
            if (afterSchedSlotsUsed < slotsRequested) {
                _cluster.setStatus(topId, "Running with fewer slots than requested (" + afterSchedSlotsUsed + "/" + origRequest + ")");
            } else if (afterSchedSlotsUsed < origRequest) {
                _cluster.setStatus(topId, "Fully Scheduled (requested " + origRequest + " slots, but could only use " + afterSchedSlotsUsed + ")");
            } else {
                _cluster.setStatus(topId, "Fully Scheduled");
            }
        } else {
            _cluster.setStatus(topId, "Fully Scheduled");
        }
    }
}