Examples with TopologyDetails org.apache.storm.scheduler.TopologyDetails used on opensource projects

Example 6 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class ResourceAwareScheduler method getUsers.

/**
     * Intialize scheduling and running queues
     *
     * @param topologies
     * @param cluster
     */
private Map<String, User> getUsers(Topologies topologies, Cluster cluster) {
    Map<String, User> userMap = new HashMap<String, User>();
    Map<String, Map<String, Double>> userResourcePools = getUserResourcePools();
    LOG.debug("userResourcePools: {}", userResourcePools);
    for (TopologyDetails td : topologies.getTopologies()) {
        String topologySubmitter = td.getTopologySubmitter();
        //additional safety check to make sure that topologySubmitter is going to be a valid value
        if (topologySubmitter == null || topologySubmitter.equals("")) {
            LOG.error("Cannot determine user for topology {}.  Will skip scheduling this topology", td.getName());
            continue;
        }
        if (!userMap.containsKey(topologySubmitter)) {
            userMap.put(topologySubmitter, new User(topologySubmitter, userResourcePools.get(topologySubmitter)));
        }
        if (cluster.getUnassignedExecutors(td).size() > 0) {
            LOG.debug("adding td: {} to pending queue", td.getName());
            userMap.get(topologySubmitter).addTopologyToPendingQueue(td);
        } else {
            LOG.debug("adding td: {} to running queue with existing status: {}", td.getName(), cluster.getStatusMap().get(td.getId()));
            userMap.get(topologySubmitter).addTopologyToRunningQueue(td);
            if (cluster.getStatusMap().get(td.getId()) == null || cluster.getStatusMap().get(td.getId()).equals("")) {
                cluster.setStatus(td.getId(), "Fully Scheduled");
            }
        }
    }
    return userMap;
}

Example 7 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class MultitenantScheduler method schedule.

@Override
public void schedule(Topologies topologies, Cluster cluster) {
    LOG.debug("Rerunning scheduling...");
    Map<String, Node> nodeIdToNode = Node.getAllNodesFrom(cluster);
    Map<String, Number> userConf = getUserConf();
    Map<String, IsolatedPool> userPools = new HashMap<>();
    for (Map.Entry<String, Number> entry : userConf.entrySet()) {
        userPools.put(entry.getKey(), new IsolatedPool(entry.getValue().intValue()));
    }
    DefaultPool defaultPool = new DefaultPool();
    FreePool freePool = new FreePool();
    freePool.init(cluster, nodeIdToNode);
    for (IsolatedPool pool : userPools.values()) {
        pool.init(cluster, nodeIdToNode);
    }
    defaultPool.init(cluster, nodeIdToNode);
    for (TopologyDetails td : topologies.getTopologies()) {
        String user = (String) td.getConf().get(Config.TOPOLOGY_SUBMITTER_USER);
        LOG.debug("Found top {} run by user {}", td.getId(), user);
        NodePool pool = userPools.get(user);
        if (pool == null || !pool.canAdd(td)) {
            pool = defaultPool;
        }
        pool.addTopology(td);
    }
    //Now schedule all of the topologies that need to be scheduled
    for (IsolatedPool pool : userPools.values()) {
        pool.scheduleAsNeeded(freePool, defaultPool);
    }
    defaultPool.scheduleAsNeeded(freePool);
    LOG.debug("Scheduling done...");
}

Example 8 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestResourceAwareScheduler method testTopologyWorkerMaxHeapSize.

@Test
public void testTopologyWorkerMaxHeapSize() {
    // Test1: If RAS spreads executors across multiple workers based on the set limit for a worker used by the topology
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 400.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 2000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(2, 2, resourceMap);
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 4);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(defaultTopologyConf);
    config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0);
    Map<ExecutorDetails, String> executorMap1 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config1);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    topoMap.put(topology1.getId(), topology1);
    Topologies topologies = new Topologies(topoMap);
    rs.prepare(config1);
    rs.schedule(topologies, cluster);
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
    Assert.assertEquals(4, cluster.getAssignedNumWorkers(topology1));
    // Test2: test when no more workers are available due to topology worker max heap size limit but there is memory is still available
    // wordSpout2 is going to contain 5 executors that needs scheduling. Each of those executors has a memory requirement of 128.0 MB
    // The cluster contains 4 free WorkerSlots. For this topolology each worker is limited to a max heap size of 128.0
    // Thus, one executor not going to be able to get scheduled thus failing the scheduling of this topology and no executors of this topology will be scheduleded
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpout2", new TestWordSpout(), 5);
    StormTopology stormTopology2 = builder2.createTopology();
    Config config2 = new Config();
    config2.putAll(defaultTopologyConf);
    config2.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0);
    Map<ExecutorDetails, String> executorMap2 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0);
    cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config2);
    topoMap = new HashMap<>();
    topoMap.put(topology2.getId(), topology2);
    topologies = new Topologies(topoMap);
    rs.prepare(config2);
    rs.schedule(topologies, cluster);
    Assert.assertEquals("Not enough resources to schedule - 0/5 executors scheduled", cluster.getStatusMap().get(topology2.getId()));
    Assert.assertEquals(5, cluster.getUnassignedExecutors(topology2).size());
}

Example 9 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestResourceAwareScheduler method sanityTestOfScheduling.

@Test
public void sanityTestOfScheduling() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 400.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 2000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(1, 2, resourceMap);
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    TopologyDetails topology1 = TestUtilsForResourceAwareScheduler.getTopology("topology1", config, 1, 1, 1, 1, 0, 0);
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    topoMap.put(topology1.getId(), topology1);
    Topologies topologies = new Topologies(topoMap);
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment = cluster.getAssignmentById(topology1.getId());
    Set<WorkerSlot> assignedSlots = assignment.getSlots();
    Set<String> nodesIDs = new HashSet<>();
    for (WorkerSlot slot : assignedSlots) {
        nodesIDs.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors = assignment.getExecutors();
    Assert.assertEquals(1, assignedSlots.size());
    Assert.assertEquals(1, nodesIDs.size());
    Assert.assertEquals(2, executors.size());
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
}

Example 10 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestResourceAwareScheduler method TestMultipleSpoutsAndCyclicTopologies.

/**
     * Test multiple spouts and cyclic topologies
     */
@Test
public void TestMultipleSpoutsAndCyclicTopologies() {
    TopologyBuilder builder = new TopologyBuilder();
    SpoutDeclarer s1 = builder.setSpout("spout-1", new TestUtilsForResourceAwareScheduler.TestSpout(), 5);
    SpoutDeclarer s2 = builder.setSpout("spout-2", new TestUtilsForResourceAwareScheduler.TestSpout(), 5);
    BoltDeclarer b1 = builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(), 5).shuffleGrouping("spout-1").shuffleGrouping("bolt-3");
    BoltDeclarer b2 = builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(), 5).shuffleGrouping("bolt-1");
    BoltDeclarer b3 = builder.setBolt("bolt-3", new TestUtilsForResourceAwareScheduler.TestBolt(), 5).shuffleGrouping("bolt-2").shuffleGrouping("spout-2");
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(25, 1, resourceMap);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 500);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 500);
    config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    StormTopology stormTopology = builder.createTopology();
    TopologyDetails topo = new TopologyDetails("topo-1", config, stormTopology, 0, genExecsAndComps(stormTopology), 0);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo.getId(), topo);
    Topologies topologies = new Topologies(topoMap);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    Assert.assertTrue("Topo scheduled?", cluster.getAssignmentById(topo.getId()) != null);
    Assert.assertEquals("Topo all executors scheduled?", 25, cluster.getAssignmentById(topo.getId()).getExecutorToSlot().size());
}