Examples with WorkerSlot backtype.storm.scheduler.WorkerSlot used on opensource projects

Example 11 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project storm-lib by xumingming.

the class DemoScheduler method schedule.

public void schedule(Topologies topologies, Cluster cluster) {
    System.out.println("DemoScheduler: begin scheduling");
    // Gets the topology which we want to schedule
    TopologyDetails topology = topologies.getByName("special-topology");
    // make sure the special topology is submitted,
    if (topology != null) {
        boolean needsScheduling = cluster.needsScheduling(topology);
        if (!needsScheduling) {
            System.out.println("Our special topology DOES NOT NEED scheduling.");
        } else {
            System.out.println("Our special topology needs scheduling.");
            // find out all the needs-scheduling components of this topology
            Map<String, List<ExecutorDetails>> componentToExecutors = cluster.getNeedsSchedulingComponentToExecutors(topology);
            System.out.println("needs scheduling(component->executor): " + componentToExecutors);
            System.out.println("needs scheduling(executor->compoenents): " + cluster.getNeedsSchedulingExecutorToComponents(topology));
            SchedulerAssignment currentAssignment = cluster.getAssignmentById(topologies.getByName("special-topology").getId());
            if (currentAssignment != null) {
                System.out.println("current assignments: " + currentAssignment.getExecutorToSlot());
            } else {
                System.out.println("current assignments: {}");
            }
            if (!componentToExecutors.containsKey("special-spout")) {
                System.out.println("Our special-spout DOES NOT NEED scheduling.");
            } else {
                System.out.println("Our special-spout needs scheduling.");
                List<ExecutorDetails> executors = componentToExecutors.get("special-spout");
                // find out the our "special-supervisor" from the supervisor metadata
                Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
                SupervisorDetails specialSupervisor = null;
                for (SupervisorDetails supervisor : supervisors) {
                    Map meta = (Map) supervisor.getSchedulerMeta();
                    if (meta.get("name").equals("special-supervisor")) {
                        specialSupervisor = supervisor;
                        break;
                    }
                }
                // found the special supervisor
                if (specialSupervisor != null) {
                    System.out.println("Found the special-supervisor");
                    List<WorkerSlot> availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // TODO for simplicity, we free all the used slots on the supervisor.
                    if (availableSlots.isEmpty() && !executors.isEmpty()) {
                        for (Integer port : cluster.getUsedPorts(specialSupervisor)) {
                            cluster.freeSlot(new WorkerSlot(specialSupervisor.getId(), port));
                        }
                    }
                    // re-get the aviableSlots
                    availableSlots = cluster.getAvailableSlots(specialSupervisor);
                    // since it is just a demo, to keep things simple, we assign all the
                    // executors into one slot.
                    cluster.assign(availableSlots.get(0), topology.getId(), executors);
                    System.out.println("We assigned executors:" + executors + " to slot: [" + availableSlots.get(0).getNodeId() + ", " + availableSlots.get(0).getPort() + "]");
                } else {
                    System.out.println("There is no supervisor named special-supervisor!!!");
                }
            }
        }
    }
    // let system's even scheduler handle the rest scheduling work
    // you can also use your own other scheduler here, this is what
    // makes storm's scheduler composable.
    new EvenScheduler().schedule(topologies, cluster);
}

Example 12 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class DefaultPool method addTopology.

@Override
public void addTopology(TopologyDetails td) {
    String topId = td.getId();
    LOG.debug("Adding in Topology {}", topId);
    _tds.put(topId, td);
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            _nodes.add(n);
        }
    }
}

Example 13 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class Node method freeTopology.

/**
     * Frees all the slots for a topology.
     * 
     * @param topId the topology to free slots for
     * @param cluster the cluster to update
     */
public void freeTopology(String topId, Cluster cluster) {
    Set<WorkerSlot> slots = _topIdToUsedSlots.get(topId);
    if (slots == null || slots.isEmpty())
        return;
    for (WorkerSlot ws : slots) {
        cluster.freeSlot(ws);
        if (_isAlive) {
            _freeSlots.add(ws);
        }
    }
    _topIdToUsedSlots.remove(topId);
}

Example 14 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class TopologyAssign method sortSlots.

/**
     * sort slots, the purpose is to ensure that the tasks are assigned in balancing
     * 
     * @param allSlots
     * @return List<WorkerSlot>
     */
public static List<WorkerSlot> sortSlots(Set<WorkerSlot> allSlots, int needSlotNum) {
    Map<String, List<WorkerSlot>> nodeMap = new HashMap<String, List<WorkerSlot>>();
    // group by first
    for (WorkerSlot np : allSlots) {
        String node = np.getNodeId();
        List<WorkerSlot> list = nodeMap.get(node);
        if (list == null) {
            list = new ArrayList<WorkerSlot>();
            nodeMap.put(node, list);
        }
        list.add(np);
    }
    for (Entry<String, List<WorkerSlot>> entry : nodeMap.entrySet()) {
        List<WorkerSlot> ports = entry.getValue();
        Collections.sort(ports, new Comparator<WorkerSlot>() {

            @Override
            public int compare(WorkerSlot first, WorkerSlot second) {
                String firstNode = first.getNodeId();
                String secondNode = second.getNodeId();
                if (firstNode.equals(secondNode) == false) {
                    return firstNode.compareTo(secondNode);
                } else {
                    return first.getPort() - second.getPort();
                }
            }
        });
    }
    // interleave
    List<List<WorkerSlot>> splitup = new ArrayList<List<WorkerSlot>>(nodeMap.values());
    Collections.sort(splitup, new Comparator<List<WorkerSlot>>() {

        public int compare(List<WorkerSlot> o1, List<WorkerSlot> o2) {
            return o2.size() - o1.size();
        }
    });
    List<WorkerSlot> sortedFreeSlots = JStormUtils.interleave_all(splitup);
    if (sortedFreeSlots.size() <= needSlotNum) {
        return sortedFreeSlots;
    }
    // sortedFreeSlots > needSlotNum
    return sortedFreeSlots.subList(0, needSlotNum);
}

Example 15 with WorkerSlot

use of backtype.storm.scheduler.WorkerSlot in project jstorm by alibaba.

the class TaskTransfer method getConnection.

protected IConnection getConnection(int taskId) {
    IConnection conn = null;
    WorkerSlot nodePort = taskNodeport.get(taskId);
    if (nodePort == null) {
        String errormsg = "IConnection to task-" + taskId + " can't be found";
        LOG.warn("Internal transfer warn, throw tuple,", new Exception(errormsg));
    } else {
        conn = nodeportSocket.get(nodePort);
        if (conn == null) {
            String errormsg = "NodePort to" + nodePort + " can't be found";
            LOG.warn("Internal transfer warn, throw tuple,", new Exception(errormsg));
        }
    }
    return conn;
}