Search in sources :

Example 31 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestLargeCluster method createTopoDetailsArray.

/**
 * Create an array of TopologyDetails by reading serialized files for topology and configuration in the
 * resource path. Skip topologies with no executors/components.
 *
 * @param failOnParseError throw exception if there are unmatched files, otherwise ignore unmatched and read errors.
 * @return An array of TopologyDetails representing resource files.
 * @throws Exception upon error in reading topology serialized files.
 */
public static TopologyDetails[] createTopoDetailsArray(String resourcePath, boolean failOnParseError) throws Exception {
    List<TopologyDetails> topoDetailsList = new ArrayList<>();
    List<String> errors = new ArrayList<>();
    List<String> resources = getResourceFiles(resourcePath);
    Map<String, String> codeResourceMap = new TreeMap<>();
    Map<String, String> confResourceMap = new HashMap<>();
    for (String resource : resources) {
        int idxOfSlash = resource.lastIndexOf("/");
        int idxOfDash = resource.lastIndexOf("-");
        String nm = idxOfDash > idxOfSlash ? resource.substring(idxOfSlash + 1, idxOfDash) : resource.substring(idxOfSlash + 1, resource.length() - COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING.length());
        if (resource.endsWith(COMPRESSED_SERIALIZED_TOPOLOGY_FILENAME_ENDING)) {
            codeResourceMap.put(nm, resource);
        } else if (resource.endsWith(COMPRESSED_SERIALIZED_CONFIG_FILENAME_ENDING)) {
            confResourceMap.put(nm, resource);
        } else {
            LOG.info("Ignoring unsupported resource file " + resource);
        }
    }
    String[] examinedConfParams = { Config.TOPOLOGY_NAME, Config.TOPOLOGY_SCHEDULER_STRATEGY, Config.TOPOLOGY_PRIORITY, Config.TOPOLOGY_WORKERS, Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Config.TOPOLOGY_SUBMITTER_USER, Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, Config.TOPOLOGY_ACKER_RESOURCES_OFFHEAP_MEMORY_MB, Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB };
    for (String topoId : codeResourceMap.keySet()) {
        String codeResource = codeResourceMap.get(topoId);
        if (!confResourceMap.containsKey(topoId)) {
            String err = String.format("Ignoring topology file %s because of missing config file for %s", codeResource, topoId);
            errors.add(err);
            LOG.error(err);
            continue;
        }
        String confResource = confResourceMap.get(topoId);
        LOG.info("Found matching topology and config files: {}, {}", codeResource, confResource);
        StormTopology stormTopology;
        try {
            stormTopology = Utils.deserialize(getResourceAsBytes(codeResource), StormTopology.class);
        } catch (Exception ex) {
            String err = String.format("Cannot read topology from resource %s", codeResource);
            errors.add(err);
            LOG.error(err, ex);
            continue;
        }
        Map<String, Object> conf;
        try {
            conf = Utils.fromCompressedJsonConf(getResourceAsBytes(confResource));
        } catch (RuntimeException | IOException ex) {
            String err = String.format("Cannot read configuration from resource %s", confResource);
            errors.add(err);
            LOG.error(err, ex);
            continue;
        }
        // fix 0.10 conf class names
        String[] configParamsToFix = { Config.TOPOLOGY_SCHEDULER_STRATEGY, Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY };
        for (String configParam : configParamsToFix) {
            if (!conf.containsKey(configParam)) {
                continue;
            }
            String className = (String) conf.get(configParam);
            if (className.startsWith("backtype")) {
                className = className.replace("backtype", "org.apache");
                conf.put(configParam, className);
            }
        }
        // fix conf params used by ConstraintSolverStrategy
        if (!conf.containsKey(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_STATE_SEARCH)) {
            conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_MAX_STATE_SEARCH, 10_000);
        }
        if (!conf.containsKey(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH)) {
            conf.put(Config.TOPOLOGY_RAS_CONSTRAINT_MAX_STATE_SEARCH, 10_000);
        }
        if (!conf.containsKey(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER)) {
            conf.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, 1);
        }
        String topoName = (String) conf.getOrDefault(Config.TOPOLOGY_NAME, topoId);
        // conf
        StringBuilder sb = new StringBuilder("Config for " + topoId + ": ");
        for (String param : examinedConfParams) {
            Object val = conf.getOrDefault(param, "<null>");
            sb.append(param).append("=").append(val).append(", ");
        }
        LOG.info(sb.toString());
        // topo
        Map<ExecutorDetails, String> execToComp = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology);
        LOG.info("Topology \"{}\" spouts={}, bolts={}, execToComp size is {}", topoName, stormTopology.get_spouts_size(), stormTopology.get_bolts_size(), execToComp.size());
        if (execToComp.isEmpty()) {
            LOG.error("Topology \"{}\" Ignoring BAD topology with zero executors", topoName);
            continue;
        }
        int numWorkers = Integer.parseInt("" + conf.getOrDefault(Config.TOPOLOGY_WORKERS, "0"));
        TopologyDetails topo = new TopologyDetails(topoId, conf, stormTopology, numWorkers, execToComp, Time.currentTimeSecs(), "user");
        // sanity check - normally this should not fail
        topo.getUserTopolgyComponents();
        topoDetailsList.add(topo);
    }
    if (!errors.isEmpty() && failOnParseError) {
        throw new Exception("Unable to parse all serialized objects\n\t" + String.join("\n\t", errors));
    }
    return topoDetailsList.toArray(new TopologyDetails[0]);
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) StormTopology(org.apache.storm.generated.StormTopology) ArrayList(java.util.ArrayList) IOException(java.io.IOException) TreeMap(java.util.TreeMap) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) IOException(java.io.IOException)

Example 32 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestDefaultResourceAwareStrategy method testDefaultResourceAwareStrategyWithoutSettingAckerExecutors.

/**
 * test if the scheduling logic for the DefaultResourceAwareStrategy is correct
 * when topology.acker.executors.per.worker is set to different values.
 *
 * If {@link Config#TOPOLOGY_ACKER_EXECUTORS} is not set,
 * it will be calculated by Nimbus as (num of estimated worker * topology.acker.executors.per.worker).
 * In this test, {@link Config#TOPOLOGY_ACKER_EXECUTORS} is set to 2 (num of estimated workers based on topo resources usage)
 *
 * For different value for {@link Config#TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER}:
 * -1: Note we don't really set it to be -1.
 *     It is just a special case in this test that topology.acker.executors.per.worker is unset, nimbus will set to 1 by default.
 * 0:  Since {@link Config#TOPOLOGY_ACKER_EXECUTORS} is not set either, acking is disabled.
 * 1:  2 ackers in total. Distribute 1 acker per worker. With ackers being added, this topology will now need 3 workers.
 *     Then first two worker will get 1 acker and last worker get 0.
 * 2:  4 ackers in total. First two workers will get 2 acker per worker respectively.
 */
@ParameterizedTest
@ValueSource(ints = { -1, 0, 1, 2 })
public void testDefaultResourceAwareStrategyWithoutSettingAckerExecutors(int numOfAckersPerWorker) throws InvalidTopologyException {
    int spoutParallelism = 1;
    int boltParallelism = 2;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new TestSpout(), spoutParallelism);
    builder.setBolt("bolt-1", new TestBolt(), boltParallelism).shuffleGrouping("spout");
    builder.setBolt("bolt-2", new TestBolt(), boltParallelism).shuffleGrouping("bolt-1");
    builder.setBolt("bolt-3", new TestBolt(), boltParallelism).shuffleGrouping("bolt-2");
    String topoName = "testTopology";
    StormTopology stormToplogy = builder.createTopology();
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 200, 2000);
    Config conf = createClusterConfig(50, 450, 0, null);
    conf.put(Config.TOPOLOGY_PRIORITY, 0);
    conf.put(Config.TOPOLOGY_NAME, topoName);
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 2000);
    conf.put(Config.TOPOLOGY_SUBMITTER_USER, "user");
    // Parameterized test on different numOfAckersPerWorker
    if (numOfAckersPerWorker == -1) {
    // Both Config.TOPOLOGY_ACKER_EXECUTORS and Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER are not set
    // Default will be 2 (estimate num of workers) and 1 respectively
    } else {
        conf.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, numOfAckersPerWorker);
    }
    int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, stormToplogy);
    Nimbus.setUpAckerExecutorConfigs(topoName, conf, conf, estimatedNumWorker);
    conf.put(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, 250);
    conf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 50);
    TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, genExecsAndComps(StormCommon.systemTopology(conf, stormToplogy)), CURRENT_TIME, "user");
    Topologies topologies = new Topologies(topo);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(conf, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    // Ordered execs: [[6, 6], [2, 2], [4, 4], [5, 5], [1, 1], [3, 3], [0, 0], [8, 8], [7, 7]]
    // Ackers: [[8, 8], [7, 7]] (+ [[9, 9], [10, 10]] when numOfAckersPerWorker=2)
    HashSet<HashSet<ExecutorDetails>> expectedScheduling = new HashSet<>();
    if (numOfAckersPerWorker == -1 || numOfAckersPerWorker == 1) {
        // Setting topology.acker.executors = null and topology.acker.executors.per.worker = null
        // are equivalent to topology.acker.executors = null and topology.acker.executors.per.worker = 1
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3
        new ExecutorDetails(6, 6), // bolt-1
        new ExecutorDetails(2, 2), // bolt-2
        new ExecutorDetails(4, 4), // acker
        new ExecutorDetails(8, 8))));
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3
        new ExecutorDetails(5, 5), // bolt-1
        new ExecutorDetails(1, 1), // bolt-2
        new ExecutorDetails(3, 3), // acker
        new ExecutorDetails(7, 7))));
        expectedScheduling.add(new HashSet<>(Arrays.asList(// spout
        new ExecutorDetails(0, 0))));
    } else if (numOfAckersPerWorker == 0) {
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3
        new ExecutorDetails(6, 6), // bolt-1
        new ExecutorDetails(2, 2), // bolt-2
        new ExecutorDetails(4, 4), // bolt-3
        new ExecutorDetails(5, 5))));
        expectedScheduling.add(new HashSet<>(Arrays.asList(// spout
        new ExecutorDetails(0, 0), // bolt-2
        new ExecutorDetails(3, 3), // bolt-1
        new ExecutorDetails(1, 1))));
    } else if (numOfAckersPerWorker == 2) {
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3
        new ExecutorDetails(6, 6), // bolt-1
        new ExecutorDetails(2, 2), // acker
        new ExecutorDetails(7, 7), // acker
        new ExecutorDetails(8, 8))));
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-2
        new ExecutorDetails(4, 4), // bolt-3
        new ExecutorDetails(5, 5), // acker
        new ExecutorDetails(9, 9), // acker
        new ExecutorDetails(10, 10))));
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-1
        new ExecutorDetails(1, 1), // bolt-2
        new ExecutorDetails(3, 3), // spout
        new ExecutorDetails(0, 0))));
    }
    HashSet<HashSet<ExecutorDetails>> foundScheduling = new HashSet<>();
    SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id");
    for (Collection<ExecutorDetails> execs : assignment.getSlotToExecutors().values()) {
        foundScheduling.add(new HashSet<>(execs));
    }
    Assert.assertEquals(expectedScheduling, foundScheduling);
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Config(org.apache.storm.Config) StormTopology(org.apache.storm.generated.StormTopology) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) HashSet(java.util.HashSet) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 33 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestDefaultResourceAwareStrategy method testDefaultResourceAwareStrategyInFavorOfShuffle.

/**
 * test if the scheduling logic for the DefaultResourceAwareStrategy (when made by network proximity needs.) is correct
 */
@Test
public void testDefaultResourceAwareStrategyInFavorOfShuffle() throws InvalidTopologyException {
    int spoutParallelism = 1;
    int boltParallelism = 2;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new TestSpout(), spoutParallelism);
    builder.setBolt("bolt-1", new TestBolt(), boltParallelism).shuffleGrouping("spout");
    builder.setBolt("bolt-2", new TestBolt(), boltParallelism).shuffleGrouping("bolt-1");
    builder.setBolt("bolt-3", new TestBolt(), boltParallelism).shuffleGrouping("bolt-2");
    StormTopology stormToplogy = builder.createTopology();
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 200, 2000);
    Config conf = createClusterConfig(50, 250, 250, null);
    conf.put(Config.TOPOLOGY_PRIORITY, 0);
    conf.put(Config.TOPOLOGY_NAME, "testTopology");
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    conf.put(Config.TOPOLOGY_SUBMITTER_USER, "user");
    conf.put(Config.TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS, true);
    TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, genExecsAndComps(StormCommon.systemTopology(conf, stormToplogy)), CURRENT_TIME, "user");
    Topologies topologies = new Topologies(topo);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(conf, new StormMetricsRegistry());
    rs.schedule(topologies, cluster);
    // Sorted execs: [[0, 0], [2, 2], [6, 6], [4, 4], [1, 1], [5, 5], [3, 3], [7, 7]]
    // Ackers: [[7, 7]]]
    HashSet<HashSet<ExecutorDetails>> expectedScheduling = new HashSet<>();
    expectedScheduling.add(new HashSet<>(Arrays.asList(// spout
    new ExecutorDetails(0, 0), // bolt-2
    new ExecutorDetails(6, 6), // bolt-1
    new ExecutorDetails(2, 2), // acker
    new ExecutorDetails(7, 7))));
    expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3
    new ExecutorDetails(3, 3), // bolt-2
    new ExecutorDetails(5, 5), // bolt-3
    new ExecutorDetails(4, 4), // bolt-1
    new ExecutorDetails(1, 1))));
    HashSet<HashSet<ExecutorDetails>> foundScheduling = new HashSet<>();
    SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id");
    for (Collection<ExecutorDetails> execs : assignment.getSlotToExecutors().values()) {
        foundScheduling.add(new HashSet<>(execs));
    }
    Assert.assertEquals(expectedScheduling, foundScheduling);
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Config(org.apache.storm.Config) StormTopology(org.apache.storm.generated.StormTopology) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) HashSet(java.util.HashSet) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 34 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestUtilsForBlacklistScheduler method genExecsAndComps.

public static Map<ExecutorDetails, String> genExecsAndComps(StormTopology topology, int spoutParallelism, int boltParallelism) {
    Map<ExecutorDetails, String> retMap = new HashMap<>();
    int startTask = 0;
    int endTask = 1;
    for (Map.Entry<String, SpoutSpec> entry : topology.get_spouts().entrySet()) {
        for (int i = 0; i < spoutParallelism; i++) {
            retMap.put(new ExecutorDetails(startTask, endTask), entry.getKey());
            startTask++;
            endTask++;
        }
    }
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        for (int i = 0; i < boltParallelism; i++) {
            retMap.put(new ExecutorDetails(startTask, endTask), entry.getKey());
            startTask++;
            endTask++;
        }
    }
    return retMap;
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) SpoutSpec(org.apache.storm.generated.SpoutSpec) HashMap(java.util.HashMap) Bolt(org.apache.storm.generated.Bolt) BaseRichBolt(org.apache.storm.topology.base.BaseRichBolt) HashMap(java.util.HashMap) Map(java.util.Map)

Example 35 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestResourceAwareScheduler method testTopologyWorkerMaxHeapSize.

@Test
public void testTopologyWorkerMaxHeapSize() {
    // Test1: If RAS spreads executors across multiple workers based on the set limit for a worker used by the topology
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 4);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(defaultTopologyConf);
    config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0, "user");
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    Topologies topologies = new Topologies(topology1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        assertFalse(cluster.needsSchedulingRas(topology1));
        assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy"));
        assertEquals(4, cluster.getAssignedNumWorkers(topology1));
    } finally {
        rs.cleanup();
    }
    // Test2: test when no more workers are available due to topology worker max heap size limit but there is memory is still available
    // wordSpout2 is going to contain 5 executors that needs scheduling. Each of those executors has a memory requirement of 128.0 MB
    // The cluster contains 4 free WorkerSlots. For this topolology each worker is limited to a max heap size of 128.0
    // Thus, one executor not going to be able to get scheduled thus failing the scheduling of this topology and no executors of this
    // topology will be scheduled
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpout2", new TestWordSpout(), 5);
    StormTopology stormTopology2 = builder2.createTopology();
    Config config2 = new Config();
    config2.putAll(defaultTopologyConf);
    config2.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0);
    Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0, "user");
    topologies = new Topologies(topology2);
    cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config2);
    rs.prepare(config2, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        assertTrue(cluster.needsSchedulingRas(topology2));
        String status = cluster.getStatusMap().get(topology2.getId());
        assert status.startsWith("Not enough resources to schedule") : status;
        // assert status.endsWith("5 executors not scheduled") : status;
        assertEquals(5, cluster.getUnassignedExecutors(topology2).size());
    } finally {
        rs.cleanup();
    }
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) HashMap(java.util.HashMap) DaemonConfig(org.apache.storm.DaemonConfig) Config(org.apache.storm.Config) StormTopology(org.apache.storm.generated.StormTopology) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) TestWordSpout(org.apache.storm.testing.TestWordSpout) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Test(org.junit.jupiter.api.Test) PerformanceTest(org.apache.storm.testing.PerformanceTest)

Aggregations

ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)72 HashMap (java.util.HashMap)50 TopologyDetails (org.apache.storm.scheduler.TopologyDetails)42 WorkerSlot (org.apache.storm.scheduler.WorkerSlot)41 SchedulerAssignment (org.apache.storm.scheduler.SchedulerAssignment)36 ArrayList (java.util.ArrayList)35 Map (java.util.Map)34 Cluster (org.apache.storm.scheduler.Cluster)31 Config (org.apache.storm.Config)29 HashSet (java.util.HashSet)28 List (java.util.List)28 SupervisorDetails (org.apache.storm.scheduler.SupervisorDetails)28 Topologies (org.apache.storm.scheduler.Topologies)23 LinkedList (java.util.LinkedList)21 INimbus (org.apache.storm.scheduler.INimbus)21 Collection (java.util.Collection)20 StormMetricsRegistry (org.apache.storm.metric.StormMetricsRegistry)19 StormTopology (org.apache.storm.generated.StormTopology)18 TestUtilsForResourceAwareScheduler (org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler)18 ResourceMetrics (org.apache.storm.scheduler.resource.normalization.ResourceMetrics)18