Examples with SchedulerAssignment backtype.storm.scheduler.SchedulerAssignment used on opensource projects

Example 1 with SchedulerAssignment

use of backtype.storm.scheduler.SchedulerAssignment in project jstorm by alibaba.

the class IsolatedPool method addTopology.

@Override
public void addTopology(TopologyDetails td) {
    String topId = td.getId();
    LOG.debug("Adding in Topology {}", topId);
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    Set<Node> assignedNodes = new HashSet<Node>();
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            assignedNodes.add(n);
        }
    }
    _usedNodes += assignedNodes.size();
    _topologyIdToNodes.put(topId, assignedNodes);
    _tds.put(topId, td);
    if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) {
        _isolated.add(topId);
    }
}

Example 2 with SchedulerAssignment

use of backtype.storm.scheduler.SchedulerAssignment in project jstorm by alibaba.

the class IsolatedPool method canAdd.

@Override
public boolean canAdd(TopologyDetails td) {
    // Only add topologies that are not sharing nodes with other topologies
    String topId = td.getId();
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            if (n.getRunningTopologies().size() > 1) {
                return false;
            }
        }
    }
    return true;
}

Example 3 with SchedulerAssignment

use of backtype.storm.scheduler.SchedulerAssignment in project jstorm by alibaba.

the class Node method getAllNodesFrom.

public static Map<String, Node> getAllNodesFrom(Cluster cluster) {
    Map<String, Node> nodeIdToNode = new HashMap<String, Node>();
    for (SupervisorDetails sup : cluster.getSupervisors().values()) {
        // Node ID and supervisor ID are the same.
        String id = sup.getId();
        boolean isAlive = !cluster.isBlackListed(id);
        LOG.debug("Found a {} Node {} {}", new Object[] { isAlive ? "living" : "dead", id, sup.getAllPorts() });
        nodeIdToNode.put(id, new Node(id, sup.getAllPorts(), isAlive));
    }
    for (Entry<String, SchedulerAssignment> entry : cluster.getAssignments().entrySet()) {
        String topId = entry.getValue().getTopologyId();
        for (WorkerSlot ws : entry.getValue().getSlots()) {
            String id = ws.getNodeId();
            Node node = nodeIdToNode.get(id);
            if (node == null) {
                LOG.debug("Found an assigned slot on a dead supervisor {}", ws);
                node = new Node(id, null, false);
                nodeIdToNode.put(id, node);
            }
            if (!node.isAlive()) {
                // The supervisor on the node down so add an orphaned slot to hold the unsupervised worker
                node.addOrphanedSlot(ws);
            }
            if (node.assignInternal(ws, topId, true)) {
                LOG.warn("Bad scheduling state for topology [" + topId + "], the slot " + ws + " assigned to multiple workers, un-assigning everything...");
                node.free(ws, cluster, true);
            }
        }
    }
    return nodeIdToNode;
}

Example 4 with SchedulerAssignment

use of backtype.storm.scheduler.SchedulerAssignment in project storm-lib by xumingming.

the class DemoScheduler method schedule.

public void schedule(Topologies topologies, Cluster cluster) {
    System.out.println("DemoScheduler: begin scheduling");
    // Gets the topology which we want to schedule
    TopologyDetails topology = topologies.getByName("special-topology");
    // make sure the special topology is submitted,
    if (topology != null) {
        boolean needsScheduling = cluster.needsScheduling(topology);
        if (!needsScheduling) {
            System.out.println("Our special topology DOES NOT NEED scheduling.");
        } else {
            System.out.println("Our special topology needs scheduling.");
            // find out all the needs-scheduling components of this topology
            Map<String, List<ExecutorDetails>> componentToExecutors = cluster.getNeedsSchedulingComponentToExecutors(topology);
            System.out.println("needs scheduling(component->executor): " + componentToExecutors);
            System.out.println("needs scheduling(executor->compoenents): " + cluster.getNeedsSchedulingExecutorToComponents(topology));
            SchedulerAssignment currentAssignment = cluster.getAssignmentById(topologies.getByName("special-topology").getId());
            if (currentAssignment != null) {
                System.out.println("current assignments: " + currentAssignment.getExecutorToSlot());
            } else {
                System.out.println("current assignments: {}");
            }
            if (!componentToExecutors.containsKey("special-spout")) {
                System.out.println("Our special-spout DOES NOT NEED scheduling.");
            } else {
                System.out.println("Our special-spout needs scheduling.");
                List<ExecutorDetails> executors = componentToExecutors.get("special-spout");
                // find out the our "special-supervisor" from the supervisor metadata
                Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
                SupervisorDetails specialSupervisor = null;
                for (SupervisorDetails supervisor : supervisors) {
                    Map meta = (Map) supervisor.getSchedulerMeta();
                    if (meta.get("name").equals("special-supervisor")) {
                        specialSupervisor = supervisor;
                        break;
                    }
                }
                // found the special supervisor
                if (specialSupervisor != null) {
                    System.out.println("Found the special-supervisor");
                    List<WorkerSlot> availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // TODO for simplicity, we free all the used slots on the supervisor.
                    if (availableSlots.isEmpty() && !executors.isEmpty()) {
                        for (Integer port : cluster.getUsedPorts(specialSupervisor)) {
                            cluster.freeSlot(new WorkerSlot(specialSupervisor.getId(), port));
                        }
                    }
                    // re-get the aviableSlots
                    availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // since it is just a demo, to keep things simple, we assign all the
                    // executors into one slot.
                    cluster.assign(availableSlots.get(0), topology.getId(), executors);
                    System.out.println("We assigned executors:" + executors + " to slot: [" + availableSlots.get(0).getNodeId() + ", " + availableSlots.get(0).getPort() + "]");
                } else {
                    System.out.println("There is no supervisor named special-supervisor!!!");
                }
            }
        }
    }
    // let system's even scheduler handle the rest scheduling work
    // you can also use your own other scheduler here, this is what
    // makes storm's scheduler composable.
    new EvenScheduler().schedule(topologies, cluster);
}

Example 5 with SchedulerAssignment

use of backtype.storm.scheduler.SchedulerAssignment in project jstorm by alibaba.

the class DefaultPool method addTopology.

@Override
public void addTopology(TopologyDetails td) {
    String topId = td.getId();
    LOG.debug("Adding in Topology {}", topId);
    _tds.put(topId, td);
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            _nodes.add(n);
        }
    }
}