use of org.apache.storm.scheduler.INimbus in project storm by apache.
the class TestResourceAwareScheduler method minCpuWorkerJustFits.
/**
* Min CPU for worker set to 50%. 1 supervisor with 100% CPU.
* A topology with 10 10% components should schedule.
*/
@Test
public void minCpuWorkerJustFits() {
INimbus iNimbus = new INimbusTest();
Map<String, SupervisorDetails> supMap = genSupervisors(1, 4, 100, 60000);
Config config = createClusterConfig(10, 500, 500, null);
config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 50.0);
TopologyDetails topo1 = genTopology("topo-1", config, 10, 0, 1, 1, currentTime - 2, 20, "jerry");
Topologies topologies = new Topologies(topo1);
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
scheduler = new ResourceAwareScheduler();
scheduler.prepare(config, new StormMetricsRegistry());
scheduler.schedule(topologies, cluster);
assertFalse(cluster.needsSchedulingRas(topo1));
assertTrue("Topo-1 scheduled?", cluster.getAssignmentById(topo1.getId()) != null);
}
use of org.apache.storm.scheduler.INimbus in project storm by apache.
the class TestResourceAwareScheduler method testSchedulingAfterFailedScheduling.
/**
* When the first topology failed to be scheduled make sure subsequent schedulings can still succeed
*/
@Test
public void testSchedulingAfterFailedScheduling() {
INimbus iNimbus = new INimbusTest();
Map<String, SupervisorDetails> supMap = genSupervisors(8, 4, 100, 1000);
Config config = createClusterConfig(100, 500, 500, null);
TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, currentTime - 2, 10, "jerry");
TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, "jerry");
TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, "jerry");
Topologies topologies = new Topologies(topo1, topo2, topo3);
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
scheduler = new ResourceAwareScheduler();
scheduler.prepare(config, new StormMetricsRegistry());
scheduler.schedule(topologies, cluster);
assertFalse("Topo-1 unscheduled?", cluster.getAssignmentById(topo1.getId()) != null);
assertTrue("Topo-2 scheduled?", cluster.getAssignmentById(topo2.getId()) != null);
assertEquals("Topo-2 all executors scheduled?", 4, cluster.getAssignmentById(topo2.getId()).getExecutorToSlot().size());
assertTrue("Topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null);
assertEquals("Topo-3 all executors scheduled?", 3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size());
}
use of org.apache.storm.scheduler.INimbus in project storm by apache.
the class TestResourceAwareScheduler method testTopologySetCpuAndMemLoad.
@Test
public void testTopologySetCpuAndMemLoad() {
INimbus iNimbus = new INimbusTest();
Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
// a topology with multiple spouts
TopologyBuilder builder1 = new TopologyBuilder();
builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0);
builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0);
StormTopology stormTopology1 = builder1.createTopology();
Config config = new Config();
config.putAll(defaultTopologyConf);
Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0, "user");
ResourceAwareScheduler rs = new ResourceAwareScheduler();
scheduler = rs;
Topologies topologies = new Topologies(topology1);
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
rs.prepare(config, new StormMetricsRegistry());
rs.schedule(topologies, cluster);
SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
Map<WorkerSlot, WorkerResources> assignedSlots1 = assignment1.getScheduledResources();
double assignedMemory = 0.0;
double assignedCpu = 0.0;
Set<String> nodesIDs1 = new HashSet<>();
for (Entry<WorkerSlot, WorkerResources> entry : assignedSlots1.entrySet()) {
WorkerResources wr = entry.getValue();
nodesIDs1.add(entry.getKey().getNodeId());
assignedMemory += wr.get_mem_on_heap() + wr.get_mem_off_heap();
assignedCpu += wr.get_cpu();
}
Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
assertEquals(1, assignedSlots1.size());
assertEquals(1, nodesIDs1.size());
assertEquals(2, executors1.size());
assertEquals(400.0, assignedMemory, 0.001);
assertEquals(40.0, assignedCpu, 0.001);
assertFalse(cluster.needsSchedulingRas(topology1));
String expectedStatusPrefix = "Running - Fully Scheduled by DefaultResourceAwareStrategy";
assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith(expectedStatusPrefix));
}
use of org.apache.storm.scheduler.INimbus in project storm by apache.
the class TestResourceAwareScheduler method testHandlingClusterSubscription.
@Test
public void testHandlingClusterSubscription() {
INimbus iNimbus = new INimbusTest();
Map<String, SupervisorDetails> supMap = genSupervisors(1, 4, 200, 1024 * 10);
Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 1_000, 8_192), userRes("bobby", 10_000, 32_768), userRes("derek", 5_000, 16_384));
Config config = createClusterConfig(10, 128, 0, resourceUserPool);
Topologies topologies = new Topologies(genTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, 20, "jerry"), genTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, 29, "jerry"));
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
scheduler = new ResourceAwareScheduler();
scheduler.prepare(config, new StormMetricsRegistry());
scheduler.schedule(topologies, cluster);
assertTopologiesFullyScheduled(cluster, "topo-1");
assertTopologiesNotScheduled(cluster, "topo-2");
}
use of org.apache.storm.scheduler.INimbus in project storm by apache.
the class TestResourceAwareScheduler method testMinCpuMaxMultipleSupervisors.
@Test
public void testMinCpuMaxMultipleSupervisors() {
INimbus iNimbus = new INimbusTest();
Map<String, SupervisorDetails> supMap = genSupervisors(3, 4, 300, 60000);
Config config = createClusterConfig(5, 50, 50, null);
config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 100.0);
TopologyDetails topo0 = genTopology("topo-0", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo1 = genTopology("topo-1", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo2 = genTopology("topo-2", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo3 = genTopology("topo-3", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo4 = genTopology("topo-4", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo5 = genTopology("topo-5", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo6 = genTopology("topo-6", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo7 = genTopology("topo-7", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo8 = genTopology("topo-8", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
TopologyDetails topo9 = genTopology("topo-9", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
Topologies topologies = new Topologies(topo0, topo1, topo2, topo3, topo4, topo5, topo6, topo7, topo8, topo9);
Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
scheduler = new ResourceAwareScheduler();
scheduler.prepare(config, new StormMetricsRegistry());
scheduler.schedule(topologies, cluster);
assertFalse(cluster.needsSchedulingRas(topo0));
assertFalse(cluster.needsSchedulingRas(topo1));
assertFalse(cluster.needsSchedulingRas(topo2));
assertFalse(cluster.needsSchedulingRas(topo3));
assertFalse(cluster.needsSchedulingRas(topo4));
assertFalse(cluster.needsSchedulingRas(topo5));
assertFalse(cluster.needsSchedulingRas(topo6));
assertFalse(cluster.needsSchedulingRas(topo7));
assertFalse(cluster.needsSchedulingRas(topo8));
assertTrue(cluster.needsSchedulingRas(topo9));
assertTrue("topo-0 scheduled?", cluster.getAssignmentById(topo0.getId()) != null);
assertTrue("topo-1 scheduled?", cluster.getAssignmentById(topo1.getId()) != null);
assertTrue("topo-2 scheduled?", cluster.getAssignmentById(topo2.getId()) != null);
assertTrue("topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null);
assertTrue("topo-4 scheduled?", cluster.getAssignmentById(topo4.getId()) != null);
assertTrue("topo-5 scheduled?", cluster.getAssignmentById(topo5.getId()) != null);
assertTrue("topo-6 scheduled?", cluster.getAssignmentById(topo6.getId()) != null);
assertTrue("topo-7 scheduled?", cluster.getAssignmentById(topo7.getId()) != null);
assertTrue("topo-8 scheduled?", cluster.getAssignmentById(topo8.getId()) != null);
assertFalse("topo-9 unscheduled?", cluster.getAssignmentById(topo9.getId()) != null);
}
Aggregations