Examples with WorkerSlot backtype.storm.scheduler.WorkerSlot used on opensource projects

Example 1 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project storm-mesos by nathanmarz.

the class MesosNimbus method toSlots.

private List<WorkerSlot> toSlots(Offer offer, int cpu, int mem) {
    OfferResources resources = getResources(offer, cpu, mem);
    List<WorkerSlot> ret = new ArrayList<WorkerSlot>();
    int availableSlots = Math.min(resources.cpuSlots, resources.memSlots);
    availableSlots = Math.min(availableSlots, resources.ports.size());
    for (int i = 0; i < availableSlots; i++) {
        ret.add(new WorkerSlot(offer.getHostname(), resources.ports.get(i)));
    }
    return ret;
}

Example 2 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project storm-lib by xumingming.

the class DemoScheduler method schedule.

public void schedule(Topologies topologies, Cluster cluster) {
    System.out.println("DemoScheduler: begin scheduling");
    // Gets the topology which we want to schedule
    TopologyDetails topology = topologies.getByName("special-topology");
    // make sure the special topology is submitted,
    if (topology != null) {
        boolean needsScheduling = cluster.needsScheduling(topology);
        if (!needsScheduling) {
            System.out.println("Our special topology DOES NOT NEED scheduling.");
        } else {
            System.out.println("Our special topology needs scheduling.");
            // find out all the needs-scheduling components of this topology
            Map<String, List<ExecutorDetails>> componentToExecutors = cluster.getNeedsSchedulingComponentToExecutors(topology);
            System.out.println("needs scheduling(component->executor): " + componentToExecutors);
            System.out.println("needs scheduling(executor->compoenents): " + cluster.getNeedsSchedulingExecutorToComponents(topology));
            SchedulerAssignment currentAssignment = cluster.getAssignmentById(topologies.getByName("special-topology").getId());
            if (currentAssignment != null) {
                System.out.println("current assignments: " + currentAssignment.getExecutorToSlot());
            } else {
                System.out.println("current assignments: {}");
            }
            if (!componentToExecutors.containsKey("special-spout")) {
                System.out.println("Our special-spout DOES NOT NEED scheduling.");
            } else {
                System.out.println("Our special-spout needs scheduling.");
                List<ExecutorDetails> executors = componentToExecutors.get("special-spout");
                // find out the our "special-supervisor" from the supervisor metadata
                Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
                SupervisorDetails specialSupervisor = null;
                for (SupervisorDetails supervisor : supervisors) {
                    Map meta = (Map) supervisor.getSchedulerMeta();
                    if (meta.get("name").equals("special-supervisor")) {
                        specialSupervisor = supervisor;
                        break;
                    }
                }
                // found the special supervisor
                if (specialSupervisor != null) {
                    System.out.println("Found the special-supervisor");
                    List<WorkerSlot> availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // TODO for simplicity, we free all the used slots on the supervisor.
                    if (availableSlots.isEmpty() && !executors.isEmpty()) {
                        for (Integer port : cluster.getUsedPorts(specialSupervisor)) {
                            cluster.freeSlot(new WorkerSlot(specialSupervisor.getId(), port));
                        }
                    }
                    // re-get the aviableSlots
                    availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // since it is just a demo, to keep things simple, we assign all the
                    // executors into one slot.
                    cluster.assign(availableSlots.get(0), topology.getId(), executors);
                    System.out.println("We assigned executors:" + executors + " to slot: [" + availableSlots.get(0).getNodeId() + ", " + availableSlots.get(0).getPort() + "]");
                } else {
                    System.out.println("There is no supervisor named special-supervisor!!!");
                }
            }
        }
    }
    // let system's even scheduler handle the rest scheduling work
    // you can also use your own other scheduler here, this is what
    // makes storm's scheduler composable.
    new EvenScheduler().schedule(topologies, cluster);
}

Example 3 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project storm-mesos by nathanmarz.

the class MesosNimbus method allSlotsAvailableForScheduling.

@Override
public Collection<WorkerSlot> allSlotsAvailableForScheduling(Collection<SupervisorDetails> existingSupervisors, Topologies topologies, Set<String> topologiesMissingAssignments) {
    synchronized (OFFERS_LOCK) {
        LOG.info("Currently have " + _offers.size() + " offers buffered");
        if (!topologiesMissingAssignments.isEmpty()) {
            LOG.info("Topologies that need assignments: " + topologiesMissingAssignments.toString());
        }
    }
    Integer cpu = null;
    Integer mem = null;
    // it will mess up scheduling on this cluster permanently
    for (String id : topologiesMissingAssignments) {
        TopologyDetails details = topologies.getById(id);
        int tcpu = MesosCommon.topologyCpu(_conf, details);
        int tmem = MesosCommon.topologyMem(_conf, details);
        if (cpu == null || tcpu > cpu) {
            cpu = tcpu;
        }
        if (mem == null || tmem > mem) {
            mem = tmem;
        }
    }
    // need access to how many slots are currently used to limit number of slots taken up
    List<WorkerSlot> allSlots = new ArrayList();
    if (cpu != null && mem != null) {
        synchronized (OFFERS_LOCK) {
            for (Offer offer : _offers.values()) {
                allSlots.addAll(toSlots(offer, cpu, mem));
            }
        }
    }
    LOG.info("Number of available slots: " + allSlots.size());
    return allSlots;
}

Example 4 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class Node method assign.

/**
 * Assign a free slot on the node to the following topology and executors. This will update the cluster too.
 *
 * @param topId     the topology to assign a free slot to.
 * @param executors the executors to run in that slot.
 * @param cluster   the cluster to be updated
 */
public void assign(String topId, Collection<ExecutorDetails> executors, Cluster cluster) {
    if (!_isAlive) {
        throw new IllegalStateException("Trying to adding to a dead node " + _nodeId);
    }
    if (_freeSlots.isEmpty()) {
        throw new IllegalStateException("Trying to assign to a full node " + _nodeId);
    }
    if (executors.size() == 0) {
        LOG.warn("Trying to assign nothing from " + topId + " to " + _nodeId + " (Ignored)");
    } else {
        WorkerSlot slot = _freeSlots.iterator().next();
        cluster.assign(slot, topId, executors);
        assignInternal(slot, topId, false);
    }
}

Example 5 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class Node method getAllNodesFrom.

public static Map<String, Node> getAllNodesFrom(Cluster cluster) {
    Map<String, Node> nodeIdToNode = new HashMap<>();
    for (SupervisorDetails sup : cluster.getSupervisors().values()) {
        // Node ID and supervisor ID are the same.
        String id = sup.getId();
        boolean isAlive = !cluster.isBlackListed(id);
        LOG.debug("Found a {} Node {} {}", new Object[] { isAlive ? "living" : "dead", id, sup.getAllPorts() });
        nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive));
    }
    for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments().entrySet()) {
        String topId = entry.getValue().getTopologyId();
        for (WorkerSlot ws : entry.getValue().getSlots()) {
            String id = ws.getNodeId();
            Node node = nodeIdToNode.get(id);
            if (node == null) {
                LOG.debug("Found an assigned slot on a dead supervisor {}", ws);
                node = new Node(id, null, false);
                nodeIdToNode.put(id, node);
            }
            if (!node.isAlive()) {
                // The supervisor on the node down so add an orphaned slot to hold the unsupervised worker
                node.addOrphanedSlot(ws);
            }
            if (node.assignInternal(ws, topId, true)) {
                LOG.warn("Bad scheduling state for topology [" + topId + "], the slot " + ws + " assigned to multiple workers, un-assigning everything...");
                node.free(ws, cluster, true);
            }
        }
    }
    return nodeIdToNode;
}