Examples with ResourceMetrics org.apache.storm.scheduler.resource.normalization.ResourceMetrics used on opensource projects

Example 31 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestBlacklistScheduler method removeLongTimeDisappearFromCache.

@Test
public void removeLongTimeDisappearFromCache() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler());
    scheduler = bs;
    bs.prepare(config, metricsRegistry);
    bs.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    for (int i = 0; i < 20; i++) {
        bs.schedule(topologies, cluster);
    }
    Set<String> cached = new HashSet<>();
    cached.add("sup-1");
    cached.add("sup-2");
    Assert.assertEquals(cached, bs.cachedSupervisors.keySet());
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    bs.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    for (int i = 0; i < 20; i++) {
        bs.schedule(topologies, cluster);
    }
    Set<Integer> cachedPorts = Sets.newHashSet(1, 2, 3);
    Assert.assertEquals(cachedPorts, bs.cachedSupervisors.get("sup-0"));
}

Example 32 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestBlacklistScheduler method blacklistSupervisorWithAddedPort.

@Test
public void blacklistSupervisorWithAddedPort() {
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 10);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    // allow blacklist scheduler to cache the supervisor
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.addPortToSupervisors(supMap, "sup-0", 4), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    // allow blacklist scheduler to cache the supervisor with an added port
    scheduler.schedule(topologies, cluster);
    // remove the port from the supervisor and make sure the blacklist scheduler can remove the port without
    // throwing an exception
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 4), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
}

Example 33 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestBlacklistScheduler method TestBadSlot.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void TestBadSlot(boolean blacklistOnBadSlot) {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT, blacklistOnBadSlot);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler.schedule(topologies, cluster);
    if (blacklistOnBadSlot) {
        Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    } else {
        Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
    }
}

Example 34 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestBlacklistScheduler method TestGreylist.

@Test
public void TestGreylist() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(2, 3);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 0.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0);
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, getDefaultResourceAwareStrategyClass().getName());
    config.put(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, true);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 1, 1, 1, 1, currentTime - 2, true);
    TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 1, 1, 1, 1, currentTime - 8, true);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new ResourceAwareScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    topoMap.put(topo1.getId(), topo1);
    topoMap.put(topo2.getId(), topo2);
    topologies = new Topologies(topoMap);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
    Assert.assertEquals("greylist", Collections.singletonList("sup-0"), cluster.getGreyListedSupervisors());
    LOG.debug("Now only these slots remain available: {}", cluster.getAvailableSlots());
    Assert.assertTrue(cluster.getAvailableSlots(supMap.get("sup-0")).containsAll(cluster.getAvailableSlots()));
}

Example 35 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestBlacklistScheduler method TestBadSupervisor.

@Test
public void TestBadSupervisor() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
}