Examples with TopologyDetails org.apache.storm.scheduler.TopologyDetails used on opensource projects

Example 11 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestUser method testMoveTopoFromPendingToRunning.

@Test
public void testMoveTopoFromPendingToRunning() {
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    List<TopologyDetails> topos = TestUtilsForResourceAwareScheduler.getListOfTopologies(config);
    User user1 = new User("user1");
    for (TopologyDetails topo : topos) {
        user1.addTopologyToPendingQueue(topo);
    }
    int counter = 1;
    for (TopologyDetails topo : topos) {
        user1.moveTopoFromPendingToRunning(topo);
        Assert.assertEquals("check correct size", (topos.size() - counter), user1.getTopologiesPending().size());
        Assert.assertEquals("check correct size", counter, user1.getTopologiesRunning().size());
        counter++;
    }
}

Example 12 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestUser method testResourcePoolUtilization.

@Test
public void testResourcePoolUtilization() {
    Double cpuGuarantee = 400.0;
    Double memoryGuarantee = 1000.0;
    Map<String, Double> resourceGuaranteeMap = new HashMap<String, Double>();
    resourceGuaranteeMap.put("cpu", cpuGuarantee);
    resourceGuaranteeMap.put("memory", memoryGuarantee);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 200);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 200);
    TopologyDetails topo1 = TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 1, 2, 1, Time.currentTimeSecs() - 24, 9);
    User user1 = new User("user1", resourceGuaranteeMap);
    user1.addTopologyToRunningQueue(topo1);
    Assert.assertEquals("check cpu resource guarantee", cpuGuarantee, user1.getCPUResourceGuaranteed(), 0.001);
    Assert.assertEquals("check memory resource guarantee", memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001);
    Assert.assertEquals("check cpu resource pool utilization", ((100.0 * 3.0) / cpuGuarantee), user1.getCPUResourcePoolUtilization(), 0.001);
    Assert.assertEquals("check memory resource pool utilization", ((200.0 + 200.0) * 3.0) / memoryGuarantee, user1.getMemoryResourcePoolUtilization(), 0.001);
}

Example 13 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestUtilsForResourceAwareScheduler method getSupervisorToCpuUsage.

public static Map<SupervisorDetails, Double> getSupervisorToCpuUsage(Cluster cluster, Topologies topologies) {
    Map<SupervisorDetails, Double> superToCpu = new HashMap<>();
    Collection<SchedulerAssignment> assignments = cluster.getAssignments().values();
    Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
    for (SupervisorDetails supervisor : supervisors) {
        superToCpu.put(supervisor, 0.0);
    }
    for (SchedulerAssignment assignment : assignments) {
        Map<ExecutorDetails, SupervisorDetails> executorToSupervisor = new HashMap<>();
        Map<SupervisorDetails, List<ExecutorDetails>> supervisorToExecutors = new HashMap<>();
        TopologyDetails topology = topologies.getById(assignment.getTopologyId());
        for (Map.Entry<ExecutorDetails, WorkerSlot> entry : assignment.getExecutorToSlot().entrySet()) {
            executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId()));
        }
        for (Map.Entry<ExecutorDetails, SupervisorDetails> entry : executorToSupervisor.entrySet()) {
            List<ExecutorDetails> executorsOnSupervisor = supervisorToExecutors.get(entry.getValue());
            if (executorsOnSupervisor == null) {
                executorsOnSupervisor = new ArrayList<>();
                supervisorToExecutors.put(entry.getValue(), executorsOnSupervisor);
            }
            executorsOnSupervisor.add(entry.getKey());
        }
        for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
            Double supervisorUsedCpu = 0.0;
            for (ExecutorDetails executor : entry.getValue()) {
                supervisorUsedCpu += topology.getTotalCpuReqTask(executor);
            }
            superToCpu.put(entry.getKey(), superToCpu.get(entry.getKey()) + supervisorUsedCpu);
        }
    }
    return superToCpu;
}

Example 14 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestUtilsForResourceAwareScheduler method getTopology.

public static TopologyDetails getTopology(String name, Map config, int numSpout, int numBolt, int spoutParallelism, int boltParallelism, int launchTime, int priority) {
    Config conf = new Config();
    conf.putAll(config);
    conf.put(Config.TOPOLOGY_PRIORITY, priority);
    conf.put(Config.TOPOLOGY_NAME, name);
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    StormTopology topology = buildTopology(numSpout, numBolt, spoutParallelism, boltParallelism);
    TopologyDetails topo = new TopologyDetails(name + "-" + launchTime, conf, topology, 0, genExecsAndComps(topology), launchTime);
    return topo;
}

Example 15 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestUtilsForResourceAwareScheduler method getSupervisorToMemoryUsage.

public static Map<SupervisorDetails, Double> getSupervisorToMemoryUsage(Cluster cluster, Topologies topologies) {
    Map<SupervisorDetails, Double> superToMem = new HashMap<>();
    Collection<SchedulerAssignment> assignments = cluster.getAssignments().values();
    Collection<SupervisorDetails> supervisors = cluster.getSupervisors().values();
    for (SupervisorDetails supervisor : supervisors) {
        superToMem.put(supervisor, 0.0);
    }
    for (SchedulerAssignment assignment : assignments) {
        Map<ExecutorDetails, SupervisorDetails> executorToSupervisor = new HashMap<>();
        Map<SupervisorDetails, List<ExecutorDetails>> supervisorToExecutors = new HashMap<>();
        TopologyDetails topology = topologies.getById(assignment.getTopologyId());
        for (Map.Entry<ExecutorDetails, WorkerSlot> entry : assignment.getExecutorToSlot().entrySet()) {
            executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId()));
        }
        for (Map.Entry<ExecutorDetails, SupervisorDetails> entry : executorToSupervisor.entrySet()) {
            List<ExecutorDetails> executorsOnSupervisor = supervisorToExecutors.get(entry.getValue());
            if (executorsOnSupervisor == null) {
                executorsOnSupervisor = new ArrayList<>();
                supervisorToExecutors.put(entry.getValue(), executorsOnSupervisor);
            }
            executorsOnSupervisor.add(entry.getKey());
        }
        for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
            Double supervisorUsedMemory = 0.0;
            for (ExecutorDetails executor : entry.getValue()) {
                supervisorUsedMemory += topology.getTotalMemReqTask(executor);
            }
            superToMem.put(entry.getKey(), superToMem.get(entry.getKey()) + supervisorUsedMemory);
        }
    }
    return superToMem;
}