Examples with ResourceAwareScheduler org.apache.storm.scheduler.resource.ResourceAwareScheduler used on opensource projects

Example 1 with ResourceAwareScheduler

use of org.apache.storm.scheduler.resource.ResourceAwareScheduler in project storm by apache.

the class TestDefaultEvictionStrategy method TestEvictMultipleTopologiesFromMultipleUsersInCorrectOrder.

/**
     * Eviction order:
     * topo-3: since user bobby don't have any resource guarantees and topo-3 is the lowest priority for user bobby
     * topo-2: since user bobby don't have any resource guarantees and topo-2 is the next lowest priority for user bobby
     * topo-5: since user derek has exceeded his resource guarantee while user jerry has not.  topo-5 and topo-4 has the same priority
     * but topo-4 was submitted earlier thus we choose that one to evict
     */
@Test
public void TestEvictMultipleTopologiesFromMultipleUsersInCorrectOrder() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 500);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 500);
    Map<String, Map<String, Number>> resourceUserPool = new HashMap<String, Map<String, Number>>();
    resourceUserPool.put("jerry", new HashMap<String, Number>());
    resourceUserPool.get("jerry").put("cpu", 300.0);
    resourceUserPool.get("jerry").put("memory", 3000.0);
    resourceUserPool.put("derek", new HashMap<String, Number>());
    resourceUserPool.get("derek").put("cpu", 100.0);
    resourceUserPool.get("derek").put("memory", 1000.0);
    config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS, resourceUserPool);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
    TopologyDetails topo1 = TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 10);
    TopologyDetails topo6 = TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 10);
    TopologyDetails topo7 = TestUtilsForResourceAwareScheduler.getTopology("topo-7", config, 1, 0, 1, 0, currentTime - 2, 10);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
    TopologyDetails topo2 = TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 10);
    TopologyDetails topo3 = TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
    TopologyDetails topo4 = TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29);
    TopologyDetails topo5 = TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0, currentTime - 15, 29);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo2.getId(), topo2);
    topoMap.put(topo3.getId(), topo3);
    topoMap.put(topo4.getId(), topo4);
    topoMap.put(topo5.getId(), topo5);
    Topologies topologies = new Topologies(topoMap);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    //user jerry submits another topology
    topoMap.put(topo1.getId(), topo1);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesAttempted()) {
        Assert.assertFalse("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of attempted topologies", 1, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of running topologies", 1, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("correct topology to evict", rs.getUser("bobby").getTopologiesAttempted().iterator().next().getName(), "topo-3");
    topoMap.put(topo6.getId(), topo6);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesAttempted()) {
        Assert.assertFalse("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of attempted topologies", 2, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of running topologies", 0, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertTrue("correct topology to evict", TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-2", rs.getUser("bobby").getTopologiesAttempted()) != null);
    Assert.assertTrue("correct topology to evict", TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-3", rs.getUser("bobby").getTopologiesAttempted()) != null);
    topoMap.put(topo7.getId(), topo7);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 3, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesAttempted()) {
        Assert.assertFalse("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 1, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    Assert.assertEquals("correct topology to evict", rs.getUser("derek").getTopologiesAttempted().iterator().next().getName(), "topo-4");
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesAttempted()) {
        Assert.assertFalse("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of attempted topologies", 2, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of running topologies", 0, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertTrue("correct topology to evict", TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-2", rs.getUser("bobby").getTopologiesAttempted()) != null);
    Assert.assertTrue("correct topology to evict", TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-3", rs.getUser("bobby").getTopologiesAttempted()) != null);
}

Example 2 with ResourceAwareScheduler

use of org.apache.storm.scheduler.resource.ResourceAwareScheduler in project storm by apache.

the class TestDefaultEvictionStrategy method TestEvictTopologyFromItself.

/**
     * If topologies from other users cannot be evicted to make space
     * check if there is a topology with lower priority that can be evicted from the current user
     */
@Test
public void TestEvictTopologyFromItself() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 500);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 500);
    Map<String, Map<String, Number>> resourceUserPool = new HashMap<String, Map<String, Number>>();
    resourceUserPool.put("jerry", new HashMap<String, Number>());
    resourceUserPool.get("jerry").put("cpu", 200.0);
    resourceUserPool.get("jerry").put("memory", 2000.0);
    resourceUserPool.put("bobby", new HashMap<String, Number>());
    resourceUserPool.get("bobby").put("cpu", 100.0);
    resourceUserPool.get("bobby").put("memory", 1000.0);
    resourceUserPool.put("derek", new HashMap<String, Number>());
    resourceUserPool.get("derek").put("cpu", 100.0);
    resourceUserPool.get("derek").put("memory", 1000.0);
    config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS, resourceUserPool);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
    TopologyDetails topo1 = TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 20);
    TopologyDetails topo2 = TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20);
    TopologyDetails topo3 = TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 29);
    TopologyDetails topo4 = TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
    TopologyDetails topo5 = TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 10);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
    TopologyDetails topo6 = TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 29);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo1.getId(), topo1);
    topoMap.put(topo2.getId(), topo2);
    topoMap.put(topo5.getId(), topo5);
    topoMap.put(topo6.getId(), topo6);
    Topologies topologies = new Topologies(topoMap);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    //user jerry submits another topology into a full cluster
    // topo3 should not be able to scheduled
    topoMap.put(topo3.getId(), topo3);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 1, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    //make sure that topo-3 didn't get scheduled.
    Assert.assertEquals("correct topology in attempted queue", rs.getUser("jerry").getTopologiesAttempted().iterator().next().getName(), "topo-3");
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    //user jerry submits another topology but this one should be scheduled since it has higher priority than than the
    //rest of jerry's running topologies
    topoMap.put(topo4.getId(), topo4);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 2, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    Assert.assertTrue("correct topology in attempted queue", TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-3", rs.getUser("jerry").getTopologiesAttempted()) != null);
    //Either topo-1 or topo-2 should have gotten evicted
    Assert.assertTrue("correct topology in attempted queue", ((TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-1", rs.getUser("jerry").getTopologiesAttempted())) != null) || (TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-2", rs.getUser("jerry").getTopologiesAttempted()) != null));
    //assert that topo-4 got scheduled
    Assert.assertTrue("correct topology in running queue", TestUtilsForResourceAwareScheduler.findTopologyInSetFromName("topo-4", rs.getUser("jerry").getTopologiesRunning()) != null);
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
}

Example 3 with ResourceAwareScheduler

use of org.apache.storm.scheduler.resource.ResourceAwareScheduler in project storm by apache.

the class Nimbus method getTopologyPageInfo.

@Override
public TopologyPageInfo getTopologyPageInfo(String topoId, String window, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getTopologyPageInfoCalls.mark();
        CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyPageInfo");
        String topoName = common.topoName;
        IStormClusterState state = stormClusterState;
        Assignment assignment = common.assignment;
        Map<List<Integer>, Map<String, Object>> beats = common.beats;
        Map<Integer, String> taskToComp = common.taskToComponent;
        StormTopology topology = common.topology;
        StormBase base = common.base;
        if (base == null) {
            throw new WrappedNotAliveException(topoId);
        }
        String owner = base.get_owner();
        Map<WorkerSlot, WorkerResources> workerToResources = getWorkerResourcesForTopology(topoId);
        List<WorkerSummary> workerSummaries = null;
        Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
        if (assignment != null) {
            Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
            Map<String, String> nodeToHost = assignment.get_node_host();
            for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                NodeInfo ni = entry.getValue();
                List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                exec2NodePort.put(entry.getKey(), nodePort);
            }
            workerSummaries = StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerToResources, includeSys, // this is the topology page, so we know the user is authorized
            true, null, owner);
        }
        TopologyPageInfo topoPageInfo = StatsUtil.aggTopoExecsStats(topoId, exec2NodePort, taskToComp, beats, topology, window, includeSys, state);
        if (topology.is_set_storm_version()) {
            topoPageInfo.set_storm_version(topology.get_storm_version());
        }
        Map<String, Object> topoConf = Utils.merge(conf, common.topoConf);
        addSpoutAggStats(topoPageInfo, topology, topoConf);
        addBoltAggStats(topoPageInfo, topology, topoConf, includeSys);
        if (workerSummaries != null) {
            topoPageInfo.set_workers(workerSummaries);
        }
        if (base.is_set_owner()) {
            topoPageInfo.set_owner(base.get_owner());
        }
        if (base.is_set_topology_version()) {
            topoPageInfo.set_topology_version(base.get_topology_version());
        }
        String schedStatus = idToSchedStatus.get().get(topoId);
        if (schedStatus != null) {
            topoPageInfo.set_sched_status(schedStatus);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null && underlyingScheduler instanceof ResourceAwareScheduler) {
            topoPageInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
            topoPageInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            topoPageInfo.set_requested_cpu(resources.getRequestedCpu());
            topoPageInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            topoPageInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            topoPageInfo.set_assigned_cpu(resources.getAssignedCpu());
            topoPageInfo.set_requested_shared_off_heap_memory(resources.getRequestedSharedMemOffHeap());
            topoPageInfo.set_requested_regular_off_heap_memory(resources.getRequestedNonSharedMemOffHeap());
            topoPageInfo.set_requested_shared_on_heap_memory(resources.getRequestedSharedMemOnHeap());
            topoPageInfo.set_requested_regular_on_heap_memory(resources.getRequestedNonSharedMemOnHeap());
            topoPageInfo.set_assigned_shared_off_heap_memory(resources.getAssignedSharedMemOffHeap());
            topoPageInfo.set_assigned_regular_off_heap_memory(resources.getAssignedNonSharedMemOffHeap());
            topoPageInfo.set_assigned_shared_on_heap_memory(resources.getAssignedSharedMemOnHeap());
            topoPageInfo.set_assigned_regular_on_heap_memory(resources.getAssignedNonSharedMemOnHeap());
            topoPageInfo.set_assigned_generic_resources(resources.getAssignedGenericResources());
            topoPageInfo.set_requested_generic_resources(resources.getRequestedGenericResources());
        }
        int launchTimeSecs = common.launchTimeSecs;
        topoPageInfo.set_name(topoName);
        topoPageInfo.set_status(extractStatusStr(base));
        topoPageInfo.set_uptime_secs(Time.deltaSecs(launchTimeSecs));
        topoPageInfo.set_topology_conf(JSONValue.toJSONString(topoConf));
        topoPageInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        if (base.is_set_component_debug()) {
            DebugOptions debug = base.get_component_debug().get(topoId);
            if (debug != null) {
                topoPageInfo.set_debug_options(debug);
            }
        }
        return topoPageInfo;
    } catch (Exception e) {
        LOG.warn("Get topo page info exception. (topology id='{}')", topoId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 4 with ResourceAwareScheduler

use of org.apache.storm.scheduler.resource.ResourceAwareScheduler in project storm by apache.

the class Nimbus method getTopologyInfoWithOptsImpl.

private TopologyInfo getTopologyInfoWithOptsImpl(String topoId, GetInfoOptions options) throws NotAliveException, AuthorizationException, InvalidTopologyException, Exception {
    CommonTopoInfo common = getCommonTopoInfo(topoId, "getTopologyInfo");
    if (common.base == null) {
        throw new WrappedNotAliveException(topoId);
    }
    IStormClusterState state = stormClusterState;
    NumErrorsChoice numErrChoice = Utils.OR(options.get_num_err_choice(), NumErrorsChoice.ALL);
    Map<String, List<ErrorInfo>> errors = new HashMap<>();
    for (String component : common.allComponents) {
        switch(numErrChoice) {
            case NONE:
                errors.put(component, Collections.emptyList());
                break;
            case ONE:
                List<ErrorInfo> errList = new ArrayList<>();
                ErrorInfo info = state.lastError(topoId, component);
                if (info != null) {
                    errList.add(info);
                }
                errors.put(component, errList);
                break;
            case ALL:
                errors.put(component, state.errors(topoId, component));
                break;
            default:
                LOG.warn("Got invalid NumErrorsChoice '{}'", numErrChoice);
                errors.put(component, state.errors(topoId, component));
                break;
        }
    }
    List<ExecutorSummary> summaries = new ArrayList<>();
    if (common.assignment != null) {
        for (Entry<List<Long>, NodeInfo> entry : common.assignment.get_executor_node_port().entrySet()) {
            NodeInfo ni = entry.getValue();
            ExecutorInfo execInfo = toExecInfo(entry.getKey());
            Map<String, String> nodeToHost = common.assignment.get_node_host();
            Map<String, Object> heartbeat = common.beats.get(ClientStatsUtil.convertExecutor(entry.getKey()));
            if (heartbeat == null) {
                heartbeat = Collections.emptyMap();
            }
            ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), nodeToHost.get(ni.get_node()), ni.get_port_iterator().next().intValue(), (Integer) heartbeat.getOrDefault("uptime", 0));
            // heartbeats "stats"
            Map ex = (Map) heartbeat.get("stats");
            if (ex != null) {
                ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex);
                summ.set_stats(stats);
            }
            summaries.add(summ);
        }
    }
    TopologyInfo topoInfo = new TopologyInfo(topoId, common.topoName, Time.deltaSecs(common.launchTimeSecs), summaries, extractStatusStr(common.base), errors);
    if (common.topology.is_set_storm_version()) {
        topoInfo.set_storm_version(common.topology.get_storm_version());
    }
    if (common.base.is_set_owner()) {
        topoInfo.set_owner(common.base.get_owner());
    }
    String schedStatus = idToSchedStatus.get().get(topoId);
    if (schedStatus != null) {
        topoInfo.set_sched_status(schedStatus);
    }
    TopologyResources resources = getResourcesForTopology(topoId, common.base);
    if (resources != null && underlyingScheduler instanceof ResourceAwareScheduler) {
        topoInfo.set_requested_memonheap(resources.getRequestedMemOnHeap());
        topoInfo.set_requested_memoffheap(resources.getRequestedMemOffHeap());
        topoInfo.set_requested_cpu(resources.getRequestedCpu());
        topoInfo.set_assigned_memonheap(resources.getAssignedMemOnHeap());
        topoInfo.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
        topoInfo.set_assigned_cpu(resources.getAssignedCpu());
    }
    if (common.base.is_set_component_debug()) {
        topoInfo.set_component_debug(common.base.get_component_debug());
    }
    topoInfo.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
    return topoInfo;
}

Example 5 with ResourceAwareScheduler

use of org.apache.storm.scheduler.resource.ResourceAwareScheduler in project storm by apache.

the class TestNodeSorterHostProximity method testAntiAffinityWithMultipleTopologies.

/**
 * Schedule two topologies, once with special resources and another without.
 * There are enough special resources to hold one topology with special resource ("my.gpu").
 * If the sort order is incorrect, scheduling will not succeed.
 */
@Test
public void testAntiAffinityWithMultipleTopologies() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisorsWithRacks(1, 40, 66, 0, 0, 4700, 226200, new HashMap<>());
    HashMap<String, Double> extraResources = new HashMap<>();
    extraResources.put("my.gpu", 1.0);
    supMap.putAll(genSupervisorsWithRacks(1, 40, 66, 1, 0, 4700, 226200, extraResources));
    Config config = new Config();
    config.putAll(createGrasClusterConfig(88, 775, 25, null, null));
    IScheduler scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    TopologyDetails tdSimple = genTopology("topology-simple", config, 1, 5, 100, 300, 0, 0, "user", 8192);
    // Schedule the simple topology first
    Topologies topologies = new Topologies(tdSimple);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    {
        NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, tdSimple);
        for (ExecutorDetails exec : tdSimple.getExecutors()) {
            nodeSorter.prepare(exec);
            List<ObjectResourcesItem> sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors.toList());
            String rackSummaries = StreamSupport.stream(sortedRacks.spliterator(), false).map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())).collect(Collectors.joining("\n\t"));
            NormalizedResourceRequest topoResourceRequest = tdSimple.getApproximateTotalResources();
            String topoRequest = String.format("Topo %s, approx-requested-resources %s", tdSimple.getId(), topoResourceRequest.toString());
            Assert.assertEquals(rackSummaries + "\n# of racks sorted", 2, sortedRacks.size());
            Assert.assertEquals(rackSummaries + "\nFirst rack sorted", "rack-000", sortedRacks.get(0).id);
            Assert.assertEquals(rackSummaries + "\nSecond rack sorted", "rack-001", sortedRacks.get(1).id);
        }
    }
    scheduler.schedule(topologies, cluster);
    TopologyBuilder builder = topologyBuilder(1, 5, 100, 300);
    builder.setBolt("gpu-bolt", new TestBolt(), 40).addResource("my.gpu", 1.0).shuffleGrouping("spout-0");
    TopologyDetails tdGpu = topoToTopologyDetails("topology-gpu", config, builder.createTopology(), 0, 0, "user", 8192);
    // Now schedule GPU but with the simple topology in place.
    topologies = new Topologies(tdSimple, tdGpu);
    cluster = new Cluster(cluster, topologies);
    {
        NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, tdGpu);
        for (ExecutorDetails exec : tdGpu.getExecutors()) {
            String comp = tdGpu.getComponentFromExecutor(exec);
            nodeSorter.prepare(exec);
            List<ObjectResourcesItem> sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors.toList());
            String rackSummaries = sortedRacks.stream().map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())).collect(Collectors.joining("\n\t"));
            NormalizedResourceRequest topoResourceRequest = tdSimple.getApproximateTotalResources();
            String topoRequest = String.format("Topo %s, approx-requested-resources %s", tdSimple.getId(), topoResourceRequest.toString());
            Assert.assertEquals(rackSummaries + "\n# of racks sorted", 2, sortedRacks.size());
            if (comp.equals("gpu-bolt")) {
                Assert.assertEquals(rackSummaries + "\nFirst rack sorted for " + comp, "rack-001", sortedRacks.get(0).id);
                Assert.assertEquals(rackSummaries + "\nSecond rack sorted for " + comp, "rack-000", sortedRacks.get(1).id);
            } else {
                Assert.assertEquals(rackSummaries + "\nFirst rack sorted for " + comp, "rack-000", sortedRacks.get(0).id);
                Assert.assertEquals(rackSummaries + "\nSecond rack sorted for " + comp, "rack-001", sortedRacks.get(1).id);
            }
        }
    }
    scheduler.schedule(topologies, cluster);
    Map<String, SchedulerAssignment> assignments = new TreeMap<>(cluster.getAssignments());
    assertEquals(2, assignments.size());
    Map<String, Map<String, AtomicLong>> topoPerRackCount = new HashMap<>();
    for (Map.Entry<String, SchedulerAssignment> entry : assignments.entrySet()) {
        SchedulerAssignment sa = entry.getValue();
        Map<String, AtomicLong> slotsPerRack = new TreeMap<>();
        for (WorkerSlot slot : sa.getSlots()) {
            String nodeId = slot.getNodeId();
            String rack = supervisorIdToRackName(nodeId);
            slotsPerRack.computeIfAbsent(rack, (r) -> new AtomicLong(0)).incrementAndGet();
        }
        LOG.info("{} => {}", entry.getKey(), slotsPerRack);
        topoPerRackCount.put(entry.getKey(), slotsPerRack);
    }
    Map<String, AtomicLong> simpleCount = topoPerRackCount.get("topology-simple-0");
    assertNotNull(simpleCount);
    // Because the simple topology was scheduled first we want to be sure that it didn't put anything on
    // the GPU nodes.
    // Only 1 rack is in use
    assertEquals(1, simpleCount.size());
    // r001 is the second rack with GPUs
    assertFalse(simpleCount.containsKey("r001"));
    // r000 is the first rack with no GPUs
    assertTrue(simpleCount.containsKey("r000"));
// We don't really care too much about the scheduling of topology-gpu-0, because it was scheduled.
}