Examples with INimbus org.apache.storm.scheduler.INimbus used on opensource projects

Example 51 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestLargeCluster method testLargeCluster.

/**
 * Create a large cluster, read topologies and configuration from resource directory and schedule.
 *
 * @throws Exception upon error.
 */
@Test
public void testLargeCluster() throws Exception {
    for (TEST_CLUSTER_NAME testClusterName : TEST_CLUSTER_NAME.values()) {
        LOG.info("********************************************");
        LOG.info("testLargeCluster: Start Processing cluster {}", testClusterName.getClusterName());
        String resourcePath = testClusterName.getResourcePath();
        Map<String, SupervisorDetails> supervisors = createSupervisors(testClusterName, 0);
        TopologyDetails[] topoDetailsArray = createTopoDetailsArray(resourcePath, false);
        Assert.assertTrue("No topologies found for cluster " + testClusterName.getClusterName(), topoDetailsArray.length > 0);
        Topologies topologies = new Topologies(topoDetailsArray);
        Config confWithDefaultStrategy = new Config();
        confWithDefaultStrategy.putAll(topoDetailsArray[0].getConf());
        confWithDefaultStrategy.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, DefaultResourceAwareStrategy.class.getName());
        confWithDefaultStrategy.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, TestUtilsForResourceAwareScheduler.GenSupervisorsDnsToSwitchMapping.class.getName());
        INimbus iNimbus = new INimbusTest();
        Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supervisors, new HashMap<>(), topologies, confWithDefaultStrategy);
        scheduler = new ResourceAwareScheduler();
        List<Class> classesToDebug = Arrays.asList(DefaultResourceAwareStrategy.class, GenericResourceAwareStrategy.class, ResourceAwareScheduler.class, Cluster.class);
        // switch to Level.DEBUG for verbose otherwise Level.INFO
        Level logLevel = Level.INFO;
        classesToDebug.forEach(x -> Configurator.setLevel(x.getName(), logLevel));
        long startTime = System.currentTimeMillis();
        scheduler.prepare(confWithDefaultStrategy, new StormMetricsRegistry());
        scheduler.schedule(topologies, cluster);
        long endTime = System.currentTimeMillis();
        LOG.info("Cluster={} Scheduling Time: {} topologies in {} seconds", testClusterName.getClusterName(), topoDetailsArray.length, (endTime - startTime) / 1000.0);
        for (TopologyDetails td : topoDetailsArray) {
            TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled(cluster, td.getName());
        }
        // Remove topology and reschedule it
        for (int i = 0; i < topoDetailsArray.length; i++) {
            startTime = System.currentTimeMillis();
            TopologyDetails topoDetails = topoDetailsArray[i];
            cluster.unassign(topoDetails.getId());
            LOG.info("Cluster={},  ({}) Removed topology {}", testClusterName.getClusterName(), i, topoDetails.getName());
            IScheduler rescheduler = new ResourceAwareScheduler();
            rescheduler.prepare(confWithDefaultStrategy, new StormMetricsRegistry());
            rescheduler.schedule(topologies, cluster);
            TestUtilsForResourceAwareScheduler.assertTopologiesFullyScheduled(cluster, topoDetails.getName());
            endTime = System.currentTimeMillis();
            LOG.info("Cluster={}, ({}) Scheduling Time: Removed topology {} and rescheduled in {} seconds", testClusterName.getClusterName(), i, topoDetails.getName(), (endTime - startTime) / 1000.0);
        }
        classesToDebug.forEach(x -> Configurator.setLevel(x.getName(), Level.INFO));
        LOG.info("testLargeCluster: End Processing cluster {}", testClusterName.getClusterName());
        LOG.info("********************************************");
    }
}

Example 52 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestDefaultEvictionStrategy method testEvictTopologyFromItself.

/**
 * If topologies from other users cannot be evicted to make space
 * check if there is a topology with lower priority that can be evicted from the current user
 */
@Test
public void testEvictTopologyFromItself() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 200, 2000), userRes("bobby", 100, 1000), userRes("derek", 100, 1000));
    Config config = createClusterConfig(100, 500, 500, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    LOG.info("\n\n\t\tScheduling topos 1,2,5,6");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone Scheduling...");
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-2", "topo-5", "topo-6");
    // user jerry submits another topology into a full cluster
    // topo3 should not be able to scheduled
    topologies = addTopologies(topologies, genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 29, "jerry"));
    cluster = new Cluster(cluster, topologies);
    LOG.info("\n\n\t\tScheduling topos 1,2,3,5,6");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone Scheduling...");
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-2", "topo-5", "topo-6");
    assertTopologiesNotScheduled(cluster, "topo-3");
    // user jerry submits another topology but this one should be scheduled since it has higher priority than than the
    // rest of jerry's running topologies
    topologies = addTopologies(topologies, genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "jerry"));
    cluster = new Cluster(cluster, topologies);
    LOG.info("\n\n\t\tScheduling topos 1-6");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone Scheduling...");
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-4", "topo-5", "topo-6");
    assertTopologiesNotScheduled(cluster, "topo-2", "topo-3");
}

Example 53 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestDefaultEvictionStrategy method testEvictMultipleTopologies.

@Test
public void testEvictMultipleTopologies() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 200, 2000), userRes("derek", 100, 1000));
    Config config = createClusterConfig(100, 500, 500, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    LOG.info("\n\n\t\tScheduling topos 2 to 5...");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone scheduling...");
    assertTopologiesFullyScheduled(cluster, "topo-2", "topo-3", "topo-4", "topo-5");
    // user jerry submits another topology
    topologies = addTopologies(topologies, genTopology("topo-1", config, 2, 0, 1, 0, currentTime - 2, 10, "jerry"));
    cluster = new Cluster(cluster, topologies);
    LOG.info("\n\n\t\tScheduling topos 1 to 5");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone scheduling...");
    // bobby has no guarantee so topo-2 and topo-3 evicted
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-4", "topo-5");
    assertTopologiesNotScheduled(cluster, "topo-2", "topo-3");
}

Example 54 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestGenericResourceAwareSchedulingPriorityStrategy method mkTestCluster.

private Cluster mkTestCluster(Topologies topologies, Config config) {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Double> genericResourcesOfferedMap = (Map<String, Double>) config.get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
    if (genericResourcesOfferedMap == null || genericResourcesOfferedMap.isEmpty()) {
        throw new IllegalArgumentException("Generic resources map must contain something in this test: " + TestGenericResourceAwareSchedulingPriorityStrategy.class.getName());
    }
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000, genericResourcesOfferedMap);
    return new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
}

Example 55 with INimbus

use of org.apache.storm.scheduler.INimbus in project storm by apache.

the class TestDefaultResourceAwareStrategy method testMultipleRacks.

/**
 * Test whether strategy will choose correct rack
 */
@Test
public void testMultipleRacks() {
    final Map<String, SupervisorDetails> supMap = new HashMap<>();
    final Map<String, SupervisorDetails> supMapRack0 = genSupervisors(10, 4, 0, 400, 8000);
    // generate another rack of supervisors with less resources
    final Map<String, SupervisorDetails> supMapRack1 = genSupervisors(10, 4, 10, 200, 4000);
    // generate some supervisors that are depleted of one resource
    final Map<String, SupervisorDetails> supMapRack2 = genSupervisors(10, 4, 20, 0, 8000);
    // generate some that has alot of memory but little of cpu
    final Map<String, SupervisorDetails> supMapRack3 = genSupervisors(10, 4, 30, 10, 8000 * 2 + 4000);
    // generate some that has alot of cpu but little of memory
    final Map<String, SupervisorDetails> supMapRack4 = genSupervisors(10, 4, 40, 400 + 200 + 10, 1000);
    // Generate some that have neither resource, to verify that the strategy will prioritize this last
    // Also put a generic resource with 0 value in the resources list, to verify that it doesn't affect the sorting
    final Map<String, SupervisorDetails> supMapRack5 = genSupervisors(10, 4, 50, 0.0, 0.0, Collections.singletonMap("gpu.count", 0.0));
    supMap.putAll(supMapRack0);
    supMap.putAll(supMapRack1);
    supMap.putAll(supMapRack2);
    supMap.putAll(supMapRack3);
    supMap.putAll(supMapRack4);
    supMap.putAll(supMapRack5);
    Config config = createClusterConfig(100, 500, 500, null);
    config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    INimbus iNimbus = new INimbusTest();
    // create test DNSToSwitchMapping plugin
    DNSToSwitchMapping TestNetworkTopographyPlugin = new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4, supMapRack5);
    // generate topologies
    TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    Topologies topologies = new Topologies(topo1, topo2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    List<String> supHostnames = new LinkedList<>();
    for (SupervisorDetails sup : supMap.values()) {
        supHostnames.add(sup.getHost());
    }
    Map<String, List<String>> rackToNodes = new HashMap<>();
    Map<String, String> resolvedSuperVisors = TestNetworkTopographyPlugin.resolve(supHostnames);
    for (Map.Entry<String, String> entry : resolvedSuperVisors.entrySet()) {
        String hostName = entry.getKey();
        String rack = entry.getValue();
        rackToNodes.computeIfAbsent(rack, rid -> new ArrayList<>()).add(hostName);
    }
    cluster.setNetworkTopography(rackToNodes);
    DefaultResourceAwareStrategyOld rs = new DefaultResourceAwareStrategyOld();
    rs.prepareForScheduling(cluster, topo1);
    INodeSorter nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS);
    nodeSorter.prepare(null);
    Iterable<ObjectResourcesItem> sortedRacks = nodeSorter.getSortedRacks();
    Iterator<ObjectResourcesItem> it = sortedRacks.iterator();
    // Ranked first since rack-0 has the most balanced set of resources
    Assert.assertEquals("rack-0 should be ordered first", "rack-0", it.next().id);
    // Ranked second since rack-1 has a balanced set of resources but less than rack-0
    Assert.assertEquals("rack-1 should be ordered second", "rack-1", it.next().id);
    // Ranked third since rack-4 has a lot of cpu but not a lot of memory
    Assert.assertEquals("rack-4 should be ordered third", "rack-4", it.next().id);
    // Ranked fourth since rack-3 has alot of memory but not cpu
    Assert.assertEquals("rack-3 should be ordered fourth", "rack-3", it.next().id);
    // Ranked fifth since rack-2 has not cpu resources
    Assert.assertEquals("rack-2 should be ordered fifth", "rack-2", it.next().id);
    // Ranked last since rack-5 has neither CPU nor memory available
    assertEquals("Rack-5 should be ordered sixth", "rack-5", it.next().id);
    SchedulingResult schedulingResult = rs.schedule(cluster, topo1);
    assert (schedulingResult.isSuccess());
    SchedulerAssignment assignment = cluster.getAssignmentById(topo1.getId());
    for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) {
        // make sure all workers on scheduled in rack-0
        Assert.assertEquals("assert worker scheduled on rack-0", "rack-0", resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()));
    }
    Assert.assertEquals("All executors in topo-1 scheduled", 0, cluster.getUnassignedExecutors(topo1).size());
    // Test if topology is already partially scheduled on one rack
    Iterator<ExecutorDetails> executorIterator = topo2.getExecutors().iterator();
    List<String> nodeHostnames = rackToNodes.get("rack-1");
    for (int i = 0; i < topo2.getExecutors().size() / 2; i++) {
        String nodeHostname = nodeHostnames.get(i % nodeHostnames.size());
        RasNode node = rs.hostnameToNodes(nodeHostname).get(0);
        WorkerSlot targetSlot = node.getFreeSlots().iterator().next();
        ExecutorDetails targetExec = executorIterator.next();
        // to keep track of free slots
        node.assign(targetSlot, topo2, Arrays.asList(targetExec));
    }
    rs = new DefaultResourceAwareStrategyOld();
    // schedule topo2
    schedulingResult = rs.schedule(cluster, topo2);
    assert (schedulingResult.isSuccess());
    assignment = cluster.getAssignmentById(topo2.getId());
    for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) {
        // make sure all workers on scheduled in rack-1
        Assert.assertEquals("assert worker scheduled on rack-1", "rack-1", resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()));
    }
    Assert.assertEquals("All executors in topo-2 scheduled", 0, cluster.getUnassignedExecutors(topo2).size());
}