Examples with Topologies org.apache.storm.scheduler.Topologies used on opensource projects

Example 81 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestResourceAwareScheduler method testTopologySetCpuAndMemLoad.

@Test
public void testTopologySetCpuAndMemLoad() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0);
    builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0, "user");
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    scheduler = rs;
    Topologies topologies = new Topologies(topology1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    rs.prepare(config, new StormMetricsRegistry());
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Map<WorkerSlot, WorkerResources> assignedSlots1 = assignment1.getScheduledResources();
    double assignedMemory = 0.0;
    double assignedCpu = 0.0;
    Set<String> nodesIDs1 = new HashSet<>();
    for (Entry<WorkerSlot, WorkerResources> entry : assignedSlots1.entrySet()) {
        WorkerResources wr = entry.getValue();
        nodesIDs1.add(entry.getKey().getNodeId());
        assignedMemory += wr.get_mem_on_heap() + wr.get_mem_off_heap();
        assignedCpu += wr.get_cpu();
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    assertEquals(1, assignedSlots1.size());
    assertEquals(1, nodesIDs1.size());
    assertEquals(2, executors1.size());
    assertEquals(400.0, assignedMemory, 0.001);
    assertEquals(40.0, assignedCpu, 0.001);
    assertFalse(cluster.needsSchedulingRas(topology1));
    String expectedStatusPrefix = "Running - Fully Scheduled by DefaultResourceAwareStrategy";
    assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith(expectedStatusPrefix));
}

Example 82 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestResourceAwareScheduler method testHandlingClusterSubscription.

@Test
public void testHandlingClusterSubscription() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(1, 4, 200, 1024 * 10);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 1_000, 8_192), userRes("bobby", 10_000, 32_768), userRes("derek", 5_000, 16_384));
    Config config = createClusterConfig(10, 128, 0, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, 20, "jerry"), genTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, 29, "jerry"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertTopologiesFullyScheduled(cluster, "topo-1");
    assertTopologiesNotScheduled(cluster, "topo-2");
}

Example 83 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestResourceAwareScheduler method testMinCpuMaxMultipleSupervisors.

@Test
public void testMinCpuMaxMultipleSupervisors() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(3, 4, 300, 60000);
    Config config = createClusterConfig(5, 50, 50, null);
    config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 100.0);
    TopologyDetails topo0 = genTopology("topo-0", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo1 = genTopology("topo-1", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo2 = genTopology("topo-2", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo3 = genTopology("topo-3", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo4 = genTopology("topo-4", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo5 = genTopology("topo-5", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo6 = genTopology("topo-6", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo7 = genTopology("topo-7", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo8 = genTopology("topo-8", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    TopologyDetails topo9 = genTopology("topo-9", config, 4, 5, 1, 1, currentTime - 2, 20, "jerry");
    Topologies topologies = new Topologies(topo0, topo1, topo2, topo3, topo4, topo5, topo6, topo7, topo8, topo9);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertFalse(cluster.needsSchedulingRas(topo0));
    assertFalse(cluster.needsSchedulingRas(topo1));
    assertFalse(cluster.needsSchedulingRas(topo2));
    assertFalse(cluster.needsSchedulingRas(topo3));
    assertFalse(cluster.needsSchedulingRas(topo4));
    assertFalse(cluster.needsSchedulingRas(topo5));
    assertFalse(cluster.needsSchedulingRas(topo6));
    assertFalse(cluster.needsSchedulingRas(topo7));
    assertFalse(cluster.needsSchedulingRas(topo8));
    assertTrue(cluster.needsSchedulingRas(topo9));
    assertTrue("topo-0 scheduled?", cluster.getAssignmentById(topo0.getId()) != null);
    assertTrue("topo-1 scheduled?", cluster.getAssignmentById(topo1.getId()) != null);
    assertTrue("topo-2 scheduled?", cluster.getAssignmentById(topo2.getId()) != null);
    assertTrue("topo-3 scheduled?", cluster.getAssignmentById(topo3.getId()) != null);
    assertTrue("topo-4 scheduled?", cluster.getAssignmentById(topo4.getId()) != null);
    assertTrue("topo-5 scheduled?", cluster.getAssignmentById(topo5.getId()) != null);
    assertTrue("topo-6 scheduled?", cluster.getAssignmentById(topo6.getId()) != null);
    assertTrue("topo-7 scheduled?", cluster.getAssignmentById(topo7.getId()) != null);
    assertTrue("topo-8 scheduled?", cluster.getAssignmentById(topo8.getId()) != null);
    assertFalse("topo-9 unscheduled?", cluster.getAssignmentById(topo9.getId()) != null);
}

Example 84 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestBlacklistScheduler method TestReleaseBlacklist.

@Test
public void TestReleaseBlacklist() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, true);
    TopologyDetails topo3 = TestUtilsForBlacklistScheduler.getTopology("topo-3", config, 5, 15, 1, 1, currentTime - 16, true);
    TopologyDetails topo4 = TestUtilsForBlacklistScheduler.getTopology("topo-4", config, 5, 15, 1, 1, currentTime - 32, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    topoMap.put(topo2.getId(), topo2);
    topoMap.put(topo3.getId(), topo3);
    topoMap.put(topo4.getId(), topo4);
    topologies = new Topologies(topoMap);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
}

Example 85 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestBlacklistScheduler method TestResumeBlacklist.

@Test
public void TestResumeBlacklist() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    for (int i = 0; i < 300 / 10 - 2; i++) {
        scheduler.schedule(topologies, cluster);
    }
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
}