Examples with Cluster org.apache.storm.scheduler.Cluster used on opensource projects

Example 86 with Cluster

use of org.apache.storm.scheduler.Cluster in project storm by apache.

the class TestResourceAwareScheduler method minCpuWorkerJustFits.

/**
 * Min CPU for worker set to 50%.  1 supervisor with 100% CPU.
 * A topology with 10 10% components should schedule.
 */
@Test
public void minCpuWorkerJustFits() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(1, 4, 100, 60000);
    Config config = createClusterConfig(10, 500, 500, null);
    config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 50.0);
    TopologyDetails topo1 = genTopology("topo-1", config, 10, 0, 1, 1, currentTime - 2, 20, "jerry");
    Topologies topologies = new Topologies(topo1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertFalse(cluster.needsSchedulingRas(topo1));
    assertTrue("Topo-1 scheduled?", cluster.getAssignmentById(topo1.getId()) != null);
}

Example 87 with Cluster

use of org.apache.storm.scheduler.Cluster in project storm by apache.

the class TestResourceAwareScheduler method testSchedulingAfterFailedScheduling.

/**
 * When the first topology failed to be scheduled make sure subsequent schedulings can still succeed
 */
@Test
public void testSchedulingAfterFailedScheduling() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(8, 4, 100, 1000);
    Config config = createClusterConfig(100, 500, 500, null);
    TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, currentTime - 2, 10, "jerry");
    TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, "jerry");
    TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, "jerry");
    Topologies topologies = new Topologies(topo1, topo2, topo3);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertFalse("Topo-1 unscheduled?", cluster.getAssignmentById(topo1.getId()) != null);
    assertTrue("Topo-2 scheduled?", cluster.getAssignmentById(topo2.getId()) != null);
    assertEquals("Topo-2 all executors scheduled?", 4, cluster.getAssignmentById(topo2.getId()).getExecutorToSlot().size());
    assertTrue("Topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null);
    assertEquals("Topo-3 all executors scheduled?", 3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size());
}

Example 88 with Cluster

use of org.apache.storm.scheduler.Cluster in project storm by apache.

the class TestResourceAwareScheduler method testTopologySetCpuAndMemLoad.

@Test
public void testTopologySetCpuAndMemLoad() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0);
    builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0, "user");
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    scheduler = rs;
    Topologies topologies = new Topologies(topology1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    rs.prepare(config, new StormMetricsRegistry());
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Map<WorkerSlot, WorkerResources> assignedSlots1 = assignment1.getScheduledResources();
    double assignedMemory = 0.0;
    double assignedCpu = 0.0;
    Set<String> nodesIDs1 = new HashSet<>();
    for (Entry<WorkerSlot, WorkerResources> entry : assignedSlots1.entrySet()) {
        WorkerResources wr = entry.getValue();
        nodesIDs1.add(entry.getKey().getNodeId());
        assignedMemory += wr.get_mem_on_heap() + wr.get_mem_off_heap();
        assignedCpu += wr.get_cpu();
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    assertEquals(1, assignedSlots1.size());
    assertEquals(1, nodesIDs1.size());
    assertEquals(2, executors1.size());
    assertEquals(400.0, assignedMemory, 0.001);
    assertEquals(40.0, assignedCpu, 0.001);
    assertFalse(cluster.needsSchedulingRas(topology1));
    String expectedStatusPrefix = "Running - Fully Scheduled by DefaultResourceAwareStrategy";
    assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith(expectedStatusPrefix));
}

Example 89 with Cluster

use of org.apache.storm.scheduler.Cluster in project storm by apache.

the class TestResourceAwareScheduler method testHandlingClusterSubscription.

@Test
public void testHandlingClusterSubscription() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(1, 4, 200, 1024 * 10);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 1_000, 8_192), userRes("bobby", 10_000, 32_768), userRes("derek", 5_000, 16_384));
    Config config = createClusterConfig(10, 128, 0, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, 20, "jerry"), genTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, 29, "jerry"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertTopologiesFullyScheduled(cluster, "topo-1");
    assertTopologiesNotScheduled(cluster, "topo-2");
}

Example 90 with Cluster

use of org.apache.storm.scheduler.Cluster in project storm by apache.

the class TestResourceAwareScheduler method testMinCpuMaxMultipleSupervisors.

@Test
public void testMinCpuMaxMultipleSupervisors() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(3, 4, 300, 60000);
    Config config = createClusterConfig(5, 50, 50, null);
    config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 100.0);
    TopologyDetails topo0 = genTopology("topo-0", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo1 = genTopology("topo-1", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo2 = genTopology("topo-2", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo3 = genTopology("topo-3", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo4 = genTopology("topo-4", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo5 = genTopology("topo-5", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo6 = genTopology("topo-6", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo7 = genTopology("topo-7", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo8 = genTopology("topo-8", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo9 = genTopology("topo-9", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    Topologies topologies = new Topologies(topo0, topo1, topo2, topo3, topo4, topo5, topo6, topo7, topo8, topo9);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertFalse(cluster.needsSchedulingRas(topo0));
    assertFalse(cluster.needsSchedulingRas(topo1));
    assertFalse(cluster.needsSchedulingRas(topo2));
    assertFalse(cluster.needsSchedulingRas(topo3));
    assertFalse(cluster.needsSchedulingRas(topo4));
    assertFalse(cluster.needsSchedulingRas(topo5));
    assertFalse(cluster.needsSchedulingRas(topo6));
    assertFalse(cluster.needsSchedulingRas(topo7));
    assertFalse(cluster.needsSchedulingRas(topo8));
    assertTrue(cluster.needsSchedulingRas(topo9));
    assertTrue("topo-0 scheduled?", cluster.getAssignmentById(topo0.getId()) != null);
    assertTrue("topo-1 scheduled?", cluster.getAssignmentById(topo1.getId()) != null);
    assertTrue("topo-2 scheduled?", cluster.getAssignmentById(topo2.getId()) != null);
    assertTrue("topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null);
    assertTrue("topo-4 scheduled?", cluster.getAssignmentById(topo4.getId()) != null);
    assertTrue("topo-5 scheduled?", cluster.getAssignmentById(topo5.getId()) != null);
    assertTrue("topo-6 scheduled?", cluster.getAssignmentById(topo6.getId()) != null);
    assertTrue("topo-7 scheduled?", cluster.getAssignmentById(topo7.getId()) != null);
    assertTrue("topo-8 scheduled?", cluster.getAssignmentById(topo8.getId()) != null);
    assertFalse("topo-9 unscheduled?", cluster.getAssignmentById(topo9.getId()) != null);
}