Examples with INimbus org.apache.storm.scheduler.INimbus used on opensource projects

Example 71 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestBlacklistScheduler method TestReleaseBlacklist.

@Test
public void TestReleaseBlacklist() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, 1, 1, currentTime - 8, true);
    TopologyDetails topo3 = TestUtilsForBlacklistScheduler.getTopology("topo-3", config, 5, 15, 1, 1, currentTime - 16, true);
    TopologyDetails topo4 = TestUtilsForBlacklistScheduler.getTopology("topo-4", config, 5, 15, 1, 1, currentTime - 32, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    topoMap.put(topo2.getId(), topo2);
    topoMap.put(topo3.getId(), topo3);
    topoMap.put(topo4.getId(), topo4);
    topologies = new Topologies(topoMap);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
}

Example 72 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestBlacklistScheduler method TestResumeBlacklist.

@Test
public void TestResumeBlacklist() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    for (int i = 0; i < 300 / 10 - 2; i++) {
        scheduler.schedule(topologies, cluster);
    }
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
}

Example 73 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestBlacklistScheduler method TestList.

@Test
public void TestList() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    topoMap.put(topo2.getId(), topo2);
    Topologies topologies = new Topologies(topoMap);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, new StormMetricsRegistry());
    List<Map<Integer, List<Integer>>> faultList = new ArrayList<>();
    faultList.add(new HashMap<>());
    faultList.add(ImmutableMap.of(0, ImmutableList.of(0, 1)));
    faultList.add(ImmutableMap.of(0, new ArrayList<>()));
    for (int i = 0; i < 17; i++) {
        faultList.add(new HashMap<>());
    }
    faultList.add(ImmutableMap.of(0, ImmutableList.of(0, 1)));
    faultList.add(ImmutableMap.of(1, ImmutableList.of(1)));
    for (int i = 0; i < 8; i++) {
        faultList.add(new HashMap<>());
    }
    faultList.add(ImmutableMap.of(0, ImmutableList.of(1)));
    faultList.add(ImmutableMap.of(1, ImmutableList.of(1)));
    for (int i = 0; i < 30; i++) {
        faultList.add(new HashMap<>());
    }
    List<Map<String, SupervisorDetails>> supervisorsList = FaultGenerateUtils.getSupervisorsList(3, 4, faultList);
    Cluster cluster = null;
    int count = 0;
    for (Map<String, SupervisorDetails> supervisors : supervisorsList) {
        cluster = FaultGenerateUtils.nextCluster(cluster, supervisors, iNimbus, config, topologies);
        scheduler.schedule(topologies, cluster);
        if (count == 0) {
            Set<String> hosts = new HashSet<>();
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 2) {
            Set<String> hosts = new HashSet<>();
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 3) {
            Set<String> hosts = new HashSet<>();
            hosts.add("host-0");
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 30) {
            Set<String> hosts = new HashSet<>();
            hosts.add("host-0");
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 31) {
            Set<String> hosts = new HashSet<>();
            hosts.add("host-0");
            hosts.add("host-1");
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 32) {
            Set<String> hosts = new HashSet<>();
            hosts.add("host-0");
            hosts.add("host-1");
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 60) {
            Set<String> hosts = new HashSet<>();
            hosts.add("host-0");
            hosts.add("host-1");
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 61) {
            Set<String> hosts = new HashSet<>();
            hosts.add("host-0");
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        } else if (count == 62) {
            Set<String> hosts = new HashSet<>();
            Assert.assertEquals("blacklist", hosts, cluster.getBlacklistedHosts());
        }
        count++;
    }
}

Example 74 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class Nimbus method launchServer.

private static Nimbus launchServer(Map<String, Object> conf, INimbus inimbus) throws Exception {
    StormCommon.validateDistributedMode(conf);
    validatePortAvailable(conf);
    OciUtils.validateImageInDaemonConf(conf);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    final Nimbus nimbus = new Nimbus(conf, inimbus, metricsRegistry);
    nimbus.launchServer();
    final ThriftServer server = new ThriftServer(conf, new Processor<>(nimbus), ThriftConnectionType.NIMBUS);
    metricsRegistry.startMetricsReporters(conf);
    Utils.addShutdownHookWithDelayedForceKill(() -> {
        metricsRegistry.stopMetricsReporters();
        nimbus.shutdown();
        server.stop();
    }, 10);
    if (ClientAuthUtils.areWorkerTokensEnabledServer(server, conf)) {
        nimbus.initWorkerTokenManager();
    }
    LOG.info("Starting nimbus server for storm version '{}'", STORM_VERSION);
    server.serve();
    return nimbus;
}