Examples with SupervisorDetails org.apache.storm.scheduler.SupervisorDetails used on opensource projects

Example 6 with SupervisorDetails

use of org.apache.storm.scheduler.SupervisorDetails in project storm by apache.

the class TestResourceAwareScheduler method testResourceLimitation.

@Test
public void testResourceLimitation() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 400.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 2000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(2, 2, resourceMap);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0).setMemoryLoad(1000.0, 200.0);
    builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0).setMemoryLoad(500.0, 100.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 2, executorMap1, 0);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    topoMap.put(topology1.getId(), topology1);
    Topologies topologies = new Topologies(topoMap);
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Set<WorkerSlot> assignedSlots1 = assignment1.getSlots();
    Set<String> nodesIDs1 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots1) {
        nodesIDs1.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    List<Double> assignedExecutorMemory = new ArrayList<>();
    List<Double> assignedExecutorCpu = new ArrayList<>();
    for (ExecutorDetails executor : executors1) {
        assignedExecutorMemory.add(topology1.getTotalMemReqTask(executor));
        assignedExecutorCpu.add(topology1.getTotalCpuReqTask(executor));
    }
    Collections.sort(assignedExecutorCpu);
    Collections.sort(assignedExecutorMemory);
    Map<ExecutorDetails, SupervisorDetails> executorToSupervisor = new HashMap<>();
    Map<SupervisorDetails, List<ExecutorDetails>> supervisorToExecutors = new HashMap<>();
    Map<Double, Double> cpuAvailableToUsed = new HashMap();
    Map<Double, Double> memoryAvailableToUsed = new HashMap();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : assignment1.getExecutorToSlot().entrySet()) {
        executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId()));
    }
    for (Map.Entry<ExecutorDetails, SupervisorDetails> entry : executorToSupervisor.entrySet()) {
        List<ExecutorDetails> executorsOnSupervisor = supervisorToExecutors.get(entry.getValue());
        if (executorsOnSupervisor == null) {
            executorsOnSupervisor = new ArrayList<>();
            supervisorToExecutors.put(entry.getValue(), executorsOnSupervisor);
        }
        executorsOnSupervisor.add(entry.getKey());
    }
    for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
        Double supervisorTotalCpu = entry.getKey().getTotalCPU();
        Double supervisorTotalMemory = entry.getKey().getTotalMemory();
        Double supervisorUsedCpu = 0.0;
        Double supervisorUsedMemory = 0.0;
        for (ExecutorDetails executor : entry.getValue()) {
            supervisorUsedMemory += topology1.getTotalCpuReqTask(executor);
            supervisorTotalCpu += topology1.getTotalMemReqTask(executor);
        }
        cpuAvailableToUsed.put(supervisorTotalCpu, supervisorUsedCpu);
        memoryAvailableToUsed.put(supervisorTotalMemory, supervisorUsedMemory);
    }
    // executor0 resides one one worker (on one), executor1 and executor2 on another worker (on the other node)
    Assert.assertEquals(2, assignedSlots1.size());
    Assert.assertEquals(2, nodesIDs1.size());
    Assert.assertEquals(3, executors1.size());
    Assert.assertEquals(100.0, assignedExecutorCpu.get(0), 0.001);
    Assert.assertEquals(250.0, assignedExecutorCpu.get(1), 0.001);
    Assert.assertEquals(250.0, assignedExecutorCpu.get(2), 0.001);
    Assert.assertEquals(600.0, assignedExecutorMemory.get(0), 0.001);
    Assert.assertEquals(1200.0, assignedExecutorMemory.get(1), 0.001);
    Assert.assertEquals(1200.0, assignedExecutorMemory.get(2), 0.001);
    for (Map.Entry<Double, Double> entry : memoryAvailableToUsed.entrySet()) {
        Assert.assertTrue(entry.getKey() - entry.getValue() >= 0);
    }
    for (Map.Entry<Double, Double> entry : cpuAvailableToUsed.entrySet()) {
        Assert.assertTrue(entry.getKey() - entry.getValue() >= 0);
    }
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
}

Example 7 with SupervisorDetails

use of org.apache.storm.scheduler.SupervisorDetails in project storm by apache.

the class TestResourceAwareScheduler method testTopologyWithMultipleSpouts.

@Test
public void testTopologyWithMultipleSpouts() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 400.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 2000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(2, 4, resourceMap);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 1);
    builder1.setSpout("wordSpout2", new TestWordSpout(), 1);
    builder1.setBolt("wordCountBolt1", new TestWordCounter(), 1).shuffleGrouping("wordSpout1").shuffleGrouping("wordSpout2");
    builder1.setBolt("wordCountBolt2", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1");
    builder1.setBolt("wordCountBolt3", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1");
    builder1.setBolt("wordCountBolt4", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt2");
    builder1.setBolt("wordCountBolt5", new TestWordCounter(), 1).shuffleGrouping("wordSpout2");
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0);
    // a topology with two unconnected partitions
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpoutX", new TestWordSpout(), 1);
    builder2.setSpout("wordSpoutY", new TestWordSpout(), 1);
    StormTopology stormTopology2 = builder2.createTopology();
    Map<ExecutorDetails, String> executorMap2 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config, stormTopology2, 0, executorMap2, 0);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    topoMap.put(topology1.getId(), topology1);
    topoMap.put(topology2.getId(), topology2);
    Topologies topologies = new Topologies(topoMap);
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Set<WorkerSlot> assignedSlots1 = assignment1.getSlots();
    Set<String> nodesIDs1 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots1) {
        nodesIDs1.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    Assert.assertEquals(1, assignedSlots1.size());
    Assert.assertEquals(1, nodesIDs1.size());
    Assert.assertEquals(7, executors1.size());
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
    SchedulerAssignment assignment2 = cluster.getAssignmentById(topology2.getId());
    Set<WorkerSlot> assignedSlots2 = assignment2.getSlots();
    Set<String> nodesIDs2 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots2) {
        nodesIDs2.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors2 = assignment2.getExecutors();
    Assert.assertEquals(1, assignedSlots2.size());
    Assert.assertEquals(1, nodesIDs2.size());
    Assert.assertEquals(2, executors2.size());
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology2.getId()));
}

Example 8 with SupervisorDetails

use of org.apache.storm.scheduler.SupervisorDetails in project storm by apache.

the class TestResourceAwareScheduler method testRASNodeSlotAssign.

@Test
public void testRASNodeSlotAssign() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 400.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 2000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(5, 4, resourceMap);
    Topologies topologies = new Topologies(new HashMap<String, TopologyDetails>());
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), new HashMap());
    Map<String, RAS_Node> nodes = RAS_Nodes.getAllNodesFrom(cluster, topologies);
    Assert.assertEquals(5, nodes.size());
    RAS_Node node = nodes.get("sup-0");
    Assert.assertEquals("sup-0", node.getId());
    Assert.assertTrue(node.isAlive());
    Assert.assertEquals(0, node.getRunningTopologies().size());
    Assert.assertTrue(node.isTotallyFree());
    Assert.assertEquals(4, node.totalSlotsFree());
    Assert.assertEquals(0, node.totalSlotsUsed());
    Assert.assertEquals(4, node.totalSlots());
    TopologyDetails topology1 = TestUtilsForResourceAwareScheduler.getTopology("topology1", new HashMap(), 1, 0, 2, 0, 0, 0);
    List<ExecutorDetails> executors11 = new ArrayList<>();
    executors11.add(new ExecutorDetails(1, 1));
    node.assign(node.getFreeSlots().iterator().next(), topology1, executors11);
    Assert.assertEquals(1, node.getRunningTopologies().size());
    Assert.assertFalse(node.isTotallyFree());
    Assert.assertEquals(3, node.totalSlotsFree());
    Assert.assertEquals(1, node.totalSlotsUsed());
    Assert.assertEquals(4, node.totalSlots());
    List<ExecutorDetails> executors12 = new ArrayList<>();
    executors12.add(new ExecutorDetails(2, 2));
    node.assign(node.getFreeSlots().iterator().next(), topology1, executors12);
    Assert.assertEquals(1, node.getRunningTopologies().size());
    Assert.assertFalse(node.isTotallyFree());
    Assert.assertEquals(2, node.totalSlotsFree());
    Assert.assertEquals(2, node.totalSlotsUsed());
    Assert.assertEquals(4, node.totalSlots());
    TopologyDetails topology2 = TestUtilsForResourceAwareScheduler.getTopology("topology2", new HashMap(), 1, 0, 2, 0, 0, 0);
    List<ExecutorDetails> executors21 = new ArrayList<>();
    executors21.add(new ExecutorDetails(1, 1));
    node.assign(node.getFreeSlots().iterator().next(), topology2, executors21);
    Assert.assertEquals(2, node.getRunningTopologies().size());
    Assert.assertFalse(node.isTotallyFree());
    Assert.assertEquals(1, node.totalSlotsFree());
    Assert.assertEquals(3, node.totalSlotsUsed());
    Assert.assertEquals(4, node.totalSlots());
    List<ExecutorDetails> executors22 = new ArrayList<>();
    executors22.add(new ExecutorDetails(2, 2));
    node.assign(node.getFreeSlots().iterator().next(), topology2, executors22);
    Assert.assertEquals(2, node.getRunningTopologies().size());
    Assert.assertFalse(node.isTotallyFree());
    Assert.assertEquals(0, node.totalSlotsFree());
    Assert.assertEquals(4, node.totalSlotsUsed());
    Assert.assertEquals(4, node.totalSlots());
    node.freeAllSlots();
    Assert.assertEquals(0, node.getRunningTopologies().size());
    Assert.assertTrue(node.isTotallyFree());
    Assert.assertEquals(4, node.totalSlotsFree());
    Assert.assertEquals(0, node.totalSlotsUsed());
    Assert.assertEquals(4, node.totalSlots());
}

Example 9 with SupervisorDetails

use of org.apache.storm.scheduler.SupervisorDetails in project storm by apache.

the class TestResourceAwareScheduler method TestSchedulingAfterFailedScheduling.

/**
     * When the first topology failed to be scheduled make sure subsequent schedulings can still succeed
     */
@Test
public void TestSchedulingAfterFailedScheduling() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(8, 4, resourceMap);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 500);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 500);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
    TopologyDetails topo1 = TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 8, 0, 2, 0, currentTime - 2, 10);
    TopologyDetails topo2 = TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20);
    TopologyDetails topo3 = TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo1.getId(), topo1);
    topoMap.put(topo2.getId(), topo2);
    topoMap.put(topo3.getId(), topo3);
    Topologies topologies = new Topologies(topoMap);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    Assert.assertTrue("Topo-2 scheduled?", cluster.getAssignmentById(topo2.getId()) != null);
    Assert.assertEquals("Topo-2 all executors scheduled?", 4, cluster.getAssignmentById(topo2.getId()).getExecutorToSlot().size());
    Assert.assertTrue("Topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null);
    Assert.assertEquals("Topo-3 all executors scheduled?", 3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size());
}

Example 10 with SupervisorDetails

use of org.apache.storm.scheduler.SupervisorDetails in project storm by apache.

the class TestResourceAwareScheduler method TestMultipleUsers.

@Test
public void TestMultipleUsers() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 1000.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0 * 10);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(20, 4, resourceMap);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    Map<String, Map<String, Number>> resourceUserPool = new HashMap<String, Map<String, Number>>();
    resourceUserPool.put("jerry", new HashMap<String, Number>());
    resourceUserPool.get("jerry").put("cpu", 1000);
    resourceUserPool.get("jerry").put("memory", 8192.0);
    resourceUserPool.put("bobby", new HashMap<String, Number>());
    resourceUserPool.get("bobby").put("cpu", 10000.0);
    resourceUserPool.get("bobby").put("memory", 32768);
    resourceUserPool.put("derek", new HashMap<String, Number>());
    resourceUserPool.get("derek").put("cpu", 5000.0);
    resourceUserPool.get("derek").put("memory", 16384.0);
    config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS, resourceUserPool);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
    TopologyDetails topo1 = TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, 20);
    TopologyDetails topo2 = TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, 29);
    TopologyDetails topo3 = TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 5, 15, 1, 1, currentTime - 16, 29);
    TopologyDetails topo4 = TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 5, 15, 1, 1, currentTime - 16, 20);
    TopologyDetails topo5 = TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 5, 15, 1, 1, currentTime - 24, 29);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
    TopologyDetails topo6 = TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 5, 15, 1, 1, currentTime - 2, 20);
    TopologyDetails topo7 = TestUtilsForResourceAwareScheduler.getTopology("topo-7", config, 5, 15, 1, 1, currentTime - 8, 29);
    TopologyDetails topo8 = TestUtilsForResourceAwareScheduler.getTopology("topo-8", config, 5, 15, 1, 1, currentTime - 16, 29);
    TopologyDetails topo9 = TestUtilsForResourceAwareScheduler.getTopology("topo-9", config, 5, 15, 1, 1, currentTime - 16, 20);
    TopologyDetails topo10 = TestUtilsForResourceAwareScheduler.getTopology("topo-10", config, 5, 15, 1, 1, currentTime - 24, 29);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
    TopologyDetails topo11 = TestUtilsForResourceAwareScheduler.getTopology("topo-11", config, 5, 15, 1, 1, currentTime - 2, 20);
    TopologyDetails topo12 = TestUtilsForResourceAwareScheduler.getTopology("topo-12", config, 5, 15, 1, 1, currentTime - 8, 29);
    TopologyDetails topo13 = TestUtilsForResourceAwareScheduler.getTopology("topo-13", config, 5, 15, 1, 1, currentTime - 16, 29);
    TopologyDetails topo14 = TestUtilsForResourceAwareScheduler.getTopology("topo-14", config, 5, 15, 1, 1, currentTime - 16, 20);
    TopologyDetails topo15 = TestUtilsForResourceAwareScheduler.getTopology("topo-15", config, 5, 15, 1, 1, currentTime - 24, 29);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo1.getId(), topo1);
    topoMap.put(topo2.getId(), topo2);
    topoMap.put(topo3.getId(), topo3);
    topoMap.put(topo4.getId(), topo4);
    topoMap.put(topo5.getId(), topo5);
    topoMap.put(topo6.getId(), topo6);
    topoMap.put(topo7.getId(), topo7);
    topoMap.put(topo8.getId(), topo8);
    topoMap.put(topo9.getId(), topo9);
    topoMap.put(topo10.getId(), topo10);
    topoMap.put(topo11.getId(), topo11);
    topoMap.put(topo12.getId(), topo12);
    topoMap.put(topo13.getId(), topo13);
    topoMap.put(topo14.getId(), topo14);
    topoMap.put(topo15.getId(), topo15);
    Topologies topologies = new Topologies(topoMap);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : topoMap.values()) {
        Assert.assertTrue(TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    for (User user : rs.getUserMap().values()) {
        Assert.assertEquals(user.getTopologiesPending().size(), 0);
        Assert.assertEquals(user.getTopologiesRunning().size(), 5);
    }
}