Search in sources :

Example 21 with SchedulerAssignment

use of org.apache.storm.scheduler.SchedulerAssignment in project storm by apache.

the class TestResourceAwareScheduler method testTopologySetCpuAndMemLoad.

@Test
public void testTopologySetCpuAndMemLoad() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 400.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 2000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(2, 2, resourceMap);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0);
    builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    topoMap.put(topology1.getId(), topology1);
    Topologies topologies = new Topologies(topoMap);
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Set<WorkerSlot> assignedSlots1 = assignment1.getSlots();
    double assignedMemory = 0.0;
    double assignedCpu = 0.0;
    Set<String> nodesIDs1 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots1) {
        nodesIDs1.add(slot.getNodeId());
        assignedMemory += slot.getAllocatedMemOnHeap() + slot.getAllocatedMemOffHeap();
        assignedCpu += slot.getAllocatedCpu();
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    Assert.assertEquals(1, assignedSlots1.size());
    Assert.assertEquals(1, nodesIDs1.size());
    Assert.assertEquals(2, executors1.size());
    Assert.assertEquals(400.0, assignedMemory, 0.001);
    Assert.assertEquals(40.0, assignedCpu, 0.001);
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Config(org.apache.storm.Config) StormTopology(org.apache.storm.generated.StormTopology) SchedulerAssignmentImpl(org.apache.storm.scheduler.SchedulerAssignmentImpl) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) HashSet(java.util.HashSet) TestWordCounter(org.apache.storm.testing.TestWordCounter) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) TestWordSpout(org.apache.storm.testing.TestWordSpout) Test(org.junit.Test)

Aggregations

SchedulerAssignment (org.apache.storm.scheduler.SchedulerAssignment)21 WorkerSlot (org.apache.storm.scheduler.WorkerSlot)20 HashMap (java.util.HashMap)18 ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)13 Map (java.util.Map)12 SupervisorDetails (org.apache.storm.scheduler.SupervisorDetails)12 ArrayList (java.util.ArrayList)11 TopologyDetails (org.apache.storm.scheduler.TopologyDetails)10 List (java.util.List)9 SchedulerAssignmentImpl (org.apache.storm.scheduler.SchedulerAssignmentImpl)8 Topologies (org.apache.storm.scheduler.Topologies)8 HashSet (java.util.HashSet)7 Config (org.apache.storm.Config)7 Cluster (org.apache.storm.scheduler.Cluster)7 INimbus (org.apache.storm.scheduler.INimbus)7 Test (org.junit.Test)7 ImmutableMap (com.google.common.collect.ImmutableMap)4 LinkedList (java.util.LinkedList)4 StormTopology (org.apache.storm.generated.StormTopology)4 TestWordSpout (org.apache.storm.testing.TestWordSpout)4