Examples with ResourceMetrics org.apache.storm.scheduler.resource.normalization.ResourceMetrics used on opensource projects

Example 6 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testStrategyTakingTooLong.

@Test
public void testStrategyTakingTooLong() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(8, 4, 100, 1000);
    Config config = createClusterConfig(100, 500, 500, null);
    List<String> allowedSchedulerStrategies = new ArrayList<>();
    allowedSchedulerStrategies.add(getDefaultResourceAwareStrategyClass().getName());
    allowedSchedulerStrategies.add(DefaultResourceAwareStrategyOld.class.getName());
    allowedSchedulerStrategies.add(NeverEndingSchedulingStrategy.class.getName());
    config.put(Config.NIMBUS_SCHEDULER_STRATEGY_CLASS_WHITELIST, allowedSchedulerStrategies);
    config.put(DaemonConfig.SCHEDULING_TIMEOUT_SECONDS_PER_TOPOLOGY, 30);
    TopologyDetails topo1 = genTopology("topo-1", config, 1, 0, 2, 0, currentTime - 2, 10, "jerry");
    TopologyDetails topo3 = genTopology("topo-3", config, 1, 2, 1, 1, currentTime - 2, 20, "jerry");
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, NeverEndingSchedulingStrategy.class.getName());
    TopologyDetails topo2 = genTopology("topo-2", config, 2, 0, 2, 0, currentTime - 2, 20, "jerry");
    Topologies topologies = new Topologies(topo1, topo2, topo3);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertFalse(cluster.needsSchedulingRas(topo1));
    assertTrue(cluster.needsSchedulingRas(topo2));
    assertFalse(cluster.needsSchedulingRas(topo3));
    assertTrue("Topo-1 scheduled?", cluster.getAssignmentById(topo1.getId()) != null);
    assertEquals("Topo-1 all executors scheduled?", 2, cluster.getAssignmentById(topo1.getId()).getExecutorToSlot().size());
    assertTrue("Topo-2 not scheduled", cluster.getAssignmentById(topo2.getId()) == null);
    assertEquals("Scheduling took too long for " + topo2.getId() + " using strategy " + NeverEndingSchedulingStrategy.class.getName() + " timeout after 30 seconds using config scheduling.timeout.seconds.per.topology.", cluster.getStatusMap().get(topo2.getId()));
    assertTrue("Topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null);
    assertEquals("Topo-3 all executors scheduled?", 3, cluster.getAssignmentById(topo3.getId()).getExecutorToSlot().size());
}

Example 7 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testTopologyWithMultipleSpouts.

@Test
public void testTopologyWithMultipleSpouts() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 4, 400, 2000);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 1);
    builder1.setSpout("wordSpout2", new TestWordSpout(), 1);
    builder1.setBolt("wordCountBolt1", new TestWordCounter(), 1).shuffleGrouping("wordSpout1").shuffleGrouping("wordSpout2");
    builder1.setBolt("wordCountBolt2", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1");
    builder1.setBolt("wordCountBolt3", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt1");
    builder1.setBolt("wordCountBolt4", new TestWordCounter(), 1).shuffleGrouping("wordCountBolt2");
    builder1.setBolt("wordCountBolt5", new TestWordCounter(), 1).shuffleGrouping("wordSpout2");
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0, "user");
    // a topology with two unconnected partitions
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpoutX", new TestWordSpout(), 1);
    builder2.setSpout("wordSpoutY", new TestWordSpout(), 1);
    StormTopology stormTopology2 = builder2.createTopology();
    Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config, stormTopology2, 0, executorMap2, 0, "user");
    scheduler = new ResourceAwareScheduler();
    Topologies topologies = new Topologies(topology1, topology2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Set<WorkerSlot> assignedSlots1 = assignment1.getSlots();
    Set<String> nodesIDs1 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots1) {
        nodesIDs1.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    assertEquals(1, assignedSlots1.size());
    assertEquals(1, nodesIDs1.size());
    assertEquals(7, executors1.size());
    assertFalse(cluster.needsSchedulingRas(topology1));
    assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy"));
    SchedulerAssignment assignment2 = cluster.getAssignmentById(topology2.getId());
    Set<WorkerSlot> assignedSlots2 = assignment2.getSlots();
    Set<String> nodesIDs2 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots2) {
        nodesIDs2.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors2 = assignment2.getExecutors();
    assertEquals(1, assignedSlots2.size());
    assertEquals(1, nodesIDs2.size());
    assertEquals(2, executors2.size());
    assertFalse(cluster.needsSchedulingRas(topology2));
    assertTrue(cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy"));
}

Example 8 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testMultipleUsers.

@Test
public void testMultipleUsers() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(20, 4, 1000, 1024 * 10);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 1_000, 8_192), userRes("bobby", 10_000, 32_768), userRes("derek", 5_000, 16_384));
    Config config = createClusterConfig(10, 128, 0, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, 20, "jerry"), genTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, 29, "jerry"), genTopology("topo-3", config, 5, 15, 1, 1, currentTime - 16, 29, "jerry"), genTopology("topo-4", config, 5, 15, 1, 1, currentTime - 16, 20, "jerry"), genTopology("topo-5", config, 5, 15, 1, 1, currentTime - 24, 29, "jerry"), genTopology("topo-6", config, 5, 15, 1, 1, currentTime - 2, 20, "bobby"), genTopology("topo-7", config, 5, 15, 1, 1, currentTime - 8, 29, "bobby"), genTopology("topo-8", config, 5, 15, 1, 1, currentTime - 16, 29, "bobby"), genTopology("topo-9", config, 5, 15, 1, 1, currentTime - 16, 20, "bobby"), genTopology("topo-10", config, 5, 15, 1, 1, currentTime - 24, 29, "bobby"), genTopology("topo-11", config, 5, 15, 1, 1, currentTime - 2, 20, "derek"), genTopology("topo-12", config, 5, 15, 1, 1, currentTime - 8, 29, "derek"), genTopology("topo-13", config, 5, 15, 1, 1, currentTime - 16, 29, "derek"), genTopology("topo-14", config, 5, 15, 1, 1, currentTime - 16, 20, "derek"), genTopology("topo-15", config, 5, 15, 1, 1, currentTime - 24, 29, "derek"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    for (TopologyDetails td : topologies) {
        assertTopologiesFullyScheduled(cluster, td.getName());
    }
}

Example 9 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testRASNodeSlotAssign.

@Test
public void testRASNodeSlotAssign() {
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(5, 4, 400, 2000);
    TopologyDetails topology1 = genTopology("topology1", config, 1, 0, 2, 0, 0, 0, "user");
    TopologyDetails topology2 = genTopology("topology2", config, 1, 0, 2, 0, 0, 0, "user");
    Topologies topologies = new Topologies(topology1, topology2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    Map<String, RasNode> nodes = RasNodes.getAllNodesFrom(cluster);
    assertEquals(5, nodes.size());
    RasNode node = nodes.get("r000s000");
    assertEquals("r000s000", node.getId());
    assertTrue(node.isAlive());
    assertEquals(0, node.getRunningTopologies().size());
    assertTrue(node.isTotallyFree());
    assertEquals(4, node.totalSlotsFree());
    assertEquals(0, node.totalSlotsUsed());
    assertEquals(4, node.totalSlots());
    List<ExecutorDetails> executors11 = new ArrayList<>();
    executors11.add(new ExecutorDetails(1, 1));
    node.assign(node.getFreeSlots().iterator().next(), topology1, executors11);
    assertEquals(1, node.getRunningTopologies().size());
    assertFalse(node.isTotallyFree());
    assertEquals(3, node.totalSlotsFree());
    assertEquals(1, node.totalSlotsUsed());
    assertEquals(4, node.totalSlots());
    List<ExecutorDetails> executors12 = new ArrayList<>();
    executors12.add(new ExecutorDetails(2, 2));
    node.assign(node.getFreeSlots().iterator().next(), topology1, executors12);
    assertEquals(1, node.getRunningTopologies().size());
    assertFalse(node.isTotallyFree());
    assertEquals(2, node.totalSlotsFree());
    assertEquals(2, node.totalSlotsUsed());
    assertEquals(4, node.totalSlots());
    List<ExecutorDetails> executors21 = new ArrayList<>();
    executors21.add(new ExecutorDetails(1, 1));
    node.assign(node.getFreeSlots().iterator().next(), topology2, executors21);
    assertEquals(2, node.getRunningTopologies().size());
    assertFalse(node.isTotallyFree());
    assertEquals(1, node.totalSlotsFree());
    assertEquals(3, node.totalSlotsUsed());
    assertEquals(4, node.totalSlots());
    List<ExecutorDetails> executors22 = new ArrayList<>();
    executors22.add(new ExecutorDetails(2, 2));
    node.assign(node.getFreeSlots().iterator().next(), topology2, executors22);
    assertEquals(2, node.getRunningTopologies().size());
    assertFalse(node.isTotallyFree());
    assertEquals(0, node.totalSlotsFree());
    assertEquals(4, node.totalSlotsUsed());
    assertEquals(4, node.totalSlots());
    node.freeAllSlots();
    assertEquals(0, node.getRunningTopologies().size());
    assertTrue(node.isTotallyFree());
    assertEquals(4, node.totalSlotsFree());
    assertEquals(0, node.totalSlotsUsed());
    assertEquals(4, node.totalSlots());
}

Example 10 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestUser method testResourcePoolUtilization.

@Test
public void testResourcePoolUtilization() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000);
    Double cpuGuarantee = 400.0;
    Double memoryGuarantee = 1000.0;
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("user1", cpuGuarantee, memoryGuarantee));
    Config config = createClusterConfig(100, 200, 200, resourceUserPool);
    TopologyDetails topo1 = genTopology("topo-1", config, 1, 1, 2, 1, Time.currentTimeSecs() - 24, 9, "user1");
    Topologies topologies = new Topologies(topo1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    User user1 = new User("user1", toDouble(resourceUserPool.get("user1")));
    WorkerSlot slot = cluster.getAvailableSlots().get(0);
    cluster.assign(slot, topo1.getId(), topo1.getExecutors());
    Assert.assertEquals("check cpu resource guarantee", cpuGuarantee, user1.getCpuResourceGuaranteed(), 0.001);
    Assert.assertEquals("check memory resource guarantee", memoryGuarantee, user1.getMemoryResourceGuaranteed(), 0.001);
    Assert.assertEquals("check cpu resource pool utilization", ((100.0 * 3.0) / cpuGuarantee), user1.getCpuResourcePoolUtilization(cluster), 0.001);
    Assert.assertEquals("check memory resource pool utilization", ((200.0 + 200.0) * 3.0) / memoryGuarantee, user1.getMemoryResourcePoolUtilization(cluster), 0.001);
}