Examples with Topologies org.apache.storm.scheduler.Topologies used on opensource projects

Example 41 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestUtilsForResourceAwareScheduler method assertTopologiesNotScheduled.

public static void assertTopologiesNotScheduled(Cluster cluster, String... topoNames) {
    Topologies topologies = cluster.getTopologies();
    for (String topoName : topoNames) {
        TopologyDetails td = topologies.getByName(topoName);
        assert (td != null) : topoName;
        String topoId = td.getId();
        String status = cluster.getStatus(topoId);
        assert (status != null) : topoName;
        assert (!isStatusSuccess(status)) : topoName;
        assert (cluster.getAssignmentById(topoId) == null) : topoName;
        assert (cluster.needsSchedulingRas(td)) : topoName;
    }
}

Example 42 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestUtilsForResourceAwareScheduler method assertTopologiesBeenEvicted.

public static void assertTopologiesBeenEvicted(Cluster cluster, Set<String> evictedTopologies, String... topoNames) {
    Topologies topologies = cluster.getTopologies();
    LOG.info("Evicted topos: {}", evictedTopologies);
    assert (evictedTopologies != null);
    for (String topoName : topoNames) {
        TopologyDetails td = topologies.getByName(topoName);
        assert (td != null) : topoName;
        String topoId = td.getId();
        assert (evictedTopologies.contains(topoId)) : topoName;
    }
}

Example 43 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestFIFOSchedulingPriorityStrategy method testFIFOEvictionStrategy.

@Test
public void testFIFOEvictionStrategy() {
    try (Time.SimulatedTime sim = new Time.SimulatedTime()) {
        INimbus iNimbus = new INimbusTest();
        Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100.0, 1000.0);
        Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 200.0, 2000.0));
        Config config = createClusterConfig(100, 500, 500, resourceUserPool);
        config.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, FIFOSchedulingPriorityStrategy.class.getName());
        Topologies topologies = new Topologies(genTopology("topo-1-jerry", config, 1, 0, 1, 0, Time.currentTimeSecs() - 250, 20, "jerry"), genTopology("topo-2-bobby", config, 1, 0, 1, 0, Time.currentTimeSecs() - 200, 10, "bobby"), genTopology("topo-3-bobby", config, 1, 0, 1, 0, Time.currentTimeSecs() - 300, 20, "bobby"), genTopology("topo-4-derek", config, 1, 0, 1, 0, Time.currentTimeSecs() - 201, 29, "derek"));
        Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
        ResourceAwareScheduler rs = new ResourceAwareScheduler();
        rs.prepare(config, new StormMetricsRegistry());
        try {
            rs.schedule(topologies, cluster);
            assertTopologiesFullyScheduled(cluster, "topo-1-jerry", "topo-2-bobby", "topo-3-bobby", "topo-4-derek");
            LOG.info("\n\n\t\tINSERTING topo-5");
            // new topology needs to be scheduled
            // topo-3 should be evicted since its been up the longest
            topologies = addTopologies(topologies, genTopology("topo-5-derek", config, 1, 0, 1, 0, Time.currentTimeSecs() - 15, 29, "derek"));
            cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
            rs.schedule(topologies, cluster);
            assertTopologiesFullyScheduled(cluster, "topo-1-jerry", "topo-2-bobby", "topo-4-derek", "topo-5-derek");
            assertTopologiesNotScheduled(cluster, "topo-3-bobby");
            LOG.info("\n\n\t\tINSERTING topo-6");
            // new topology needs to be scheduled.  topo-4 should be evicted. Even though topo-1 from user jerry is older, topo-1 will not be evicted
            // since user jerry has enough resource guarantee
            topologies = addTopologies(topologies, genTopology("topo-6-bobby", config, 1, 0, 1, 0, Time.currentTimeSecs() - 10, 29, "bobby"));
            cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
            rs.schedule(topologies, cluster);
            assertTopologiesFullyScheduled(cluster, "topo-1-jerry", "topo-2-bobby", "topo-5-derek", "topo-6-bobby");
            assertTopologiesNotScheduled(cluster, "topo-3-bobby", "topo-4-derek");
        } finally {
            rs.cleanup();
        }
    }
}

Example 44 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestBlacklistScheduler method removeLongTimeDisappearFromCache.

@Test
public void removeLongTimeDisappearFromCache() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    BlacklistScheduler bs = new BlacklistScheduler(new DefaultScheduler());
    scheduler = bs;
    bs.prepare(config, metricsRegistry);
    bs.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    for (int i = 0; i < 20; i++) {
        bs.schedule(topologies, cluster);
    }
    Set<String> cached = new HashSet<>();
    cached.add("sup-1");
    cached.add("sup-2");
    Assert.assertEquals(cached, bs.cachedSupervisors.keySet());
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    bs.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    for (int i = 0; i < 20; i++) {
        bs.schedule(topologies, cluster);
    }
    Set<Integer> cachedPorts = Sets.newHashSet(1, 2, 3);
    Assert.assertEquals(cachedPorts, bs.cachedSupervisors.get("sup-0"));
}

Example 45 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestBlacklistScheduler method blacklistSupervisorWithAddedPort.

@Test
public void blacklistSupervisorWithAddedPort() {
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 10);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    // allow blacklist scheduler to cache the supervisor
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.addPortToSupervisors(supMap, "sup-0", 4), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    // allow blacklist scheduler to cache the supervisor with an added port
    scheduler.schedule(topologies, cluster);
    // remove the port from the supervisor and make sure the blacklist scheduler can remove the port without
    // throwing an exception
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 4), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
}