Examples with Topologies org.apache.storm.scheduler.Topologies used on opensource projects

Example 46 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestBlacklistScheduler method TestBadSlot.

@ParameterizedTest
@ValueSource(booleans = { true, false })
public void TestBadSlot(boolean blacklistOnBadSlot) {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_ASSUME_SUPERVISOR_BAD_BASED_ON_BAD_SLOT, blacklistOnBadSlot);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removePortFromSupervisors(supMap, "sup-0", 0), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler.schedule(topologies, cluster);
    if (blacklistOnBadSlot) {
        Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    } else {
        Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
    }
}

Example 47 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestBlacklistScheduler method TestGreylist.

@Test
public void TestGreylist() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(2, 3);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 0.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0);
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, getDefaultResourceAwareStrategyClass().getName());
    config.put(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER, true);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 1, 1, 1, 1, currentTime - 2, true);
    TopologyDetails topo2 = TestUtilsForBlacklistScheduler.getTopology("topo-2", config, 1, 1, 1, 1, currentTime - 8, true);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new BlacklistScheduler(new ResourceAwareScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
    topoMap.put(topo1.getId(), topo1);
    topoMap.put(topo2.getId(), topo2);
    topologies = new Topologies(topoMap);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.emptySet(), cluster.getBlacklistedHosts());
    Assert.assertEquals("greylist", Collections.singletonList("sup-0"), cluster.getGreyListedSupervisors());
    LOG.debug("Now only these slots remain available: {}", cluster.getAvailableSlots());
    Assert.assertTrue(cluster.getAvailableSlots(supMap.get("sup-0")).containsAll(cluster.getAvailableSlots()));
}

Example 48 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class TestBlacklistScheduler method TestBadSupervisor.

@Test
public void TestBadSupervisor() {
    INimbus iNimbus = new TestUtilsForBlacklistScheduler.INimbusTest();
    Map<String, SupervisorDetails> supMap = TestUtilsForBlacklistScheduler.genSupervisors(3, 4);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_TIME, 200);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_TOLERANCE_COUNT, 2);
    config.put(DaemonConfig.BLACKLIST_SCHEDULER_RESUME_TIME, 300);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    TopologyDetails topo1 = TestUtilsForBlacklistScheduler.getTopology("topo-1", config, 5, 15, 1, 1, currentTime - 2, true);
    topoMap.put(topo1.getId(), topo1);
    Topologies topologies = new Topologies(topoMap);
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    ResourceMetrics resourceMetrics = new ResourceMetrics(metricsRegistry);
    Cluster cluster = new Cluster(iNimbus, resourceMetrics, supMap, new HashMap<>(), topologies, config);
    scheduler = new BlacklistScheduler(new DefaultScheduler());
    scheduler.prepare(config, metricsRegistry);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, TestUtilsForBlacklistScheduler.removeSupervisorFromSupervisors(supMap, "sup-0"), TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    cluster = new Cluster(iNimbus, resourceMetrics, supMap, TestUtilsForBlacklistScheduler.assignmentMapToImpl(cluster.getAssignments()), topologies, config);
    scheduler.schedule(topologies, cluster);
    Assert.assertEquals("blacklist", Collections.singleton("host-0"), cluster.getBlacklistedHosts());
}

Example 49 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class IsolationScheduler method leftoverTopologies.

private Topologies leftoverTopologies(Topologies topologies, Set<String> filterIds) {
    Collection<TopologyDetails> topos = topologies.getTopologies();
    Map<String, TopologyDetails> leftoverTopologies = new HashMap<String, TopologyDetails>();
    for (TopologyDetails topo : topos) {
        String id = topo.getId();
        if (!filterIds.contains(id)) {
            leftoverTopologies.put(id, topo);
        }
    }
    return new Topologies(leftoverTopologies);
}

Example 50 with Topologies

use of org.apache.storm.scheduler.Topologies in project storm by apache.

the class IsolationScheduler method schedule.

// get host -> all assignable worker slots for non-blacklisted machines (assigned or not assigned)
// will then have a list of machines that need to be assigned (machine -> [topology, list of list of executors])
// match each spec to a machine (who has the right number of workers), free everything else on that machine and assign those slots (do one topology at a time)
// blacklist all machines who had production slots defined
// log isolated topologies who weren't able to get enough slots / machines
// run default scheduler on isolated topologies that didn't have enough slots + non-isolated topologies on remaining machines
// set blacklist to what it was initially
@Override
public void schedule(Topologies topologies, Cluster cluster) {
    Set<String> origBlacklist = cluster.getBlacklistedHosts();
    List<TopologyDetails> isoTopologies = isolatedTopologies(topologies.getTopologies());
    Set<String> isoIds = extractTopologyIds(isoTopologies);
    Map<String, Set<Set<ExecutorDetails>>> topologyWorkerSpecs = topologyWorkerSpecs(isoTopologies);
    Map<String, Map<Integer, Integer>> topologyMachineDistributions = topologyMachineDistributions(isoTopologies);
    Map<String, List<AssignmentInfo>> hostAssignments = hostAssignments(cluster);
    for (Map.Entry<String, List<AssignmentInfo>> entry : hostAssignments.entrySet()) {
        List<AssignmentInfo> assignments = entry.getValue();
        String topologyId = assignments.get(0).getTopologyId();
        Map<Integer, Integer> distribution = topologyMachineDistributions.get(topologyId);
        Set<Set<ExecutorDetails>> workerSpecs = topologyWorkerSpecs.get(topologyId);
        int numWorkers = assignments.size();
        if (isoIds.contains(topologyId) && checkAssignmentTopology(assignments, topologyId) && distribution.containsKey(numWorkers) && checkAssignmentWorkerSpecs(assignments, workerSpecs)) {
            decrementDistribution(distribution, numWorkers);
            for (AssignmentInfo ass : assignments) {
                workerSpecs.remove(ass.getExecutors());
            }
            cluster.blacklistHost(entry.getKey());
        } else {
            for (AssignmentInfo ass : assignments) {
                if (isoIds.contains(ass.getTopologyId())) {
                    cluster.freeSlot(ass.getWorkerSlot());
                }
            }
        }
    }
    Map<String, Set<WorkerSlot>> hostUsedSlots = hostToUsedSlots(cluster);
    LinkedList<HostAssignableSlots> hss = hostAssignableSlots(cluster);
    for (Map.Entry<String, Set<Set<ExecutorDetails>>> entry : topologyWorkerSpecs.entrySet()) {
        String topologyId = entry.getKey();
        Set<Set<ExecutorDetails>> executorSet = entry.getValue();
        List<Integer> workerNum = distributionToSortedAmounts(topologyMachineDistributions.get(topologyId));
        for (Integer num : workerNum) {
            HostAssignableSlots hostSlots = hss.peek();
            List<WorkerSlot> slot = hostSlots != null ? hostSlots.getWorkerSlots() : null;
            if (slot != null && slot.size() >= num) {
                hss.poll();
                cluster.freeSlots(hostUsedSlots.get(hostSlots.getHostName()));
                for (WorkerSlot tmpSlot : slot.subList(0, num)) {
                    Set<ExecutorDetails> executor = removeElemFromExecutorsSet(executorSet);
                    cluster.assign(tmpSlot, topologyId, executor);
                }
                cluster.blacklistHost(hostSlots.getHostName());
            }
        }
    }
    List<String> failedTopologyIds = extractFailedTopologyIds(topologyWorkerSpecs);
    if (failedTopologyIds.size() > 0) {
        LOG.warn("Unable to isolate topologies " + failedTopologyIds + ". No machine had enough worker slots to run the remaining workers for these topologies. " + "Clearing all other resources and will wait for enough resources for " + "isolated topologies before allocating any other resources.");
        // clear workers off all hosts that are not blacklisted
        Map<String, Set<WorkerSlot>> usedSlots = hostToUsedSlots(cluster);
        Set<Map.Entry<String, Set<WorkerSlot>>> entries = usedSlots.entrySet();
        for (Map.Entry<String, Set<WorkerSlot>> entry : entries) {
            if (!cluster.isBlacklistedHost(entry.getKey())) {
                cluster.freeSlots(entry.getValue());
            }
        }
    } else {
        // run default scheduler on non-isolated topologies
        Set<String> allocatedTopologies = allocatedTopologies(topologyWorkerSpecs);
        Topologies leftOverTopologies = leftoverTopologies(topologies, allocatedTopologies);
        DefaultScheduler.defaultSchedule(leftOverTopologies, cluster);
    }
    cluster.setBlacklistedHosts(origBlacklist);
}