Examples with SchedulerAssignment org.apache.storm.scheduler.SchedulerAssignment used on opensource projects

Example 6 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class TestUtilsForResourceAwareScheduler method getSupervisorToCpuUsage.

public static Map<SupervisorDetails, Double> getSupervisorToCpuUsage(Cluster cluster, Topologies topologies) {
    Map<SupervisorDetails, Double> superToCpu = new HashMap<>();
    Collection<SchedulerAssignment> assignments = cluster.getAssignments().values();
    Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
    for (SupervisorDetails supervisor : supervisors) {
        superToCpu.put(supervisor, 0.0);
    }
    for (SchedulerAssignment assignment : assignments) {
        Map<ExecutorDetails, SupervisorDetails> executorToSupervisor = new HashMap<>();
        Map<SupervisorDetails, List<ExecutorDetails>> supervisorToExecutors = new HashMap<>();
        TopologyDetails topology = topologies.getById(assignment.getTopologyId());
        for (Map.Entry<ExecutorDetails, WorkerSlot> entry : assignment.getExecutorToSlot().entrySet()) {
            executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId()));
        }
        for (Map.Entry<ExecutorDetails, SupervisorDetails> entry : executorToSupervisor.entrySet()) {
            List<ExecutorDetails> executorsOnSupervisor = supervisorToExecutors.get(entry.getValue());
            if (executorsOnSupervisor == null) {
                executorsOnSupervisor = new ArrayList<>();
                supervisorToExecutors.put(entry.getValue(), executorsOnSupervisor);
            }
            executorsOnSupervisor.add(entry.getKey());
        }
        for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
            Double supervisorUsedCpu = 0.0;
            for (ExecutorDetails executor : entry.getValue()) {
                supervisorUsedCpu += topology.getTotalCpuReqTask(executor);
            }
            superToCpu.put(entry.getKey(), superToCpu.get(entry.getKey()) + supervisorUsedCpu);
        }
    }
    return superToCpu;
}

Example 7 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class TestUtilsForResourceAwareScheduler method getSupervisorToMemoryUsage.

public static Map<SupervisorDetails, Double> getSupervisorToMemoryUsage(Cluster cluster, Topologies topologies) {
    Map<SupervisorDetails, Double> superToMem = new HashMap<>();
    Collection<SchedulerAssignment> assignments = cluster.getAssignments().values();
    Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
    for (SupervisorDetails supervisor : supervisors) {
        superToMem.put(supervisor, 0.0);
    }
    for (SchedulerAssignment assignment : assignments) {
        Map<ExecutorDetails, SupervisorDetails> executorToSupervisor = new HashMap<>();
        Map<SupervisorDetails, List<ExecutorDetails>> supervisorToExecutors = new HashMap<>();
        TopologyDetails topology = topologies.getById(assignment.getTopologyId());
        for (Map.Entry<ExecutorDetails, WorkerSlot> entry : assignment.getExecutorToSlot().entrySet()) {
            executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId()));
        }
        for (Map.Entry<ExecutorDetails, SupervisorDetails> entry : executorToSupervisor.entrySet()) {
            List<ExecutorDetails> executorsOnSupervisor = supervisorToExecutors.get(entry.getValue());
            if (executorsOnSupervisor == null) {
                executorsOnSupervisor = new ArrayList<>();
                supervisorToExecutors.put(entry.getValue(), executorsOnSupervisor);
            }
            executorsOnSupervisor.add(entry.getKey());
        }
        for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
            Double supervisorUsedMemory = 0.0;
            for (ExecutorDetails executor : entry.getValue()) {
                supervisorUsedMemory += topology.getTotalMemReqTask(executor);
            }
            superToMem.put(entry.getKey(), superToMem.get(entry.getKey()) + supervisorUsedMemory);
        }
    }
    return superToMem;
}

Example 8 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class IsolationScheduler method hostAssignments.

private Map<String, List<AssignmentInfo>> hostAssignments(Cluster cluster) {
    Collection<SchedulerAssignment> assignmentValues = cluster.getAssignments().values();
    Map<String, List<AssignmentInfo>> hostAssignments = new HashMap<String, List<AssignmentInfo>>();
    for (SchedulerAssignment sa : assignmentValues) {
        Map<WorkerSlot, List<ExecutorDetails>> slotExecutors = Utils.reverseMap(sa.getExecutorToSlot());
        Set<Map.Entry<WorkerSlot, List<ExecutorDetails>>> entries = slotExecutors.entrySet();
        for (Map.Entry<WorkerSlot, List<ExecutorDetails>> entry : entries) {
            WorkerSlot slot = entry.getKey();
            List<ExecutorDetails> executors = entry.getValue();
            String host = cluster.getHost(slot.getNodeId());
            AssignmentInfo ass = new AssignmentInfo(slot, sa.getTopologyId(), new HashSet<ExecutorDetails>(executors));
            List<AssignmentInfo> executorList = hostAssignments.get(host);
            if (executorList == null) {
                executorList = new ArrayList<AssignmentInfo>();
                hostAssignments.put(host, executorList);
            }
            executorList.add(ass);
        }
    }
    return hostAssignments;
}

Example 9 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class DefaultPool method addTopology.

@Override
public void addTopology(TopologyDetails td) {
    String topId = td.getId();
    LOG.debug("Adding in Topology {}", topId);
    _tds.put(topId, td);
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            _nodes.add(n);
        }
    }
}

Example 10 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class IsolatedPool method addTopology.

@Override
public void addTopology(TopologyDetails td) {
    String topId = td.getId();
    LOG.debug("Adding in Topology {}", topId);
    SchedulerAssignment assignment = _cluster.getAssignmentById(topId);
    Set<Node> assignedNodes = new HashSet<>();
    if (assignment != null) {
        for (WorkerSlot ws : assignment.getSlots()) {
            Node n = _nodeIdToNode.get(ws.getNodeId());
            assignedNodes.add(n);
        }
    }
    _usedNodes += assignedNodes.size();
    _topologyIdToNodes.put(topId, assignedNodes);
    _tds.put(topId, td);
    if (td.getConf().get(Config.TOPOLOGY_ISOLATED_MACHINES) != null) {
        _isolated.add(topId);
    }
}