Search in sources :

Example 26 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class Nimbus method getResourcesForTopology.

private TopologyResources getResourcesForTopology(String topoId, StormBase base) throws NotAliveException, AuthorizationException, InvalidTopologyException, IOException {
    TopologyResources ret = idToResources.get().get(topoId);
    if (ret == null) {
        try {
            IStormClusterState state = stormClusterState;
            TopologyDetails details = readTopologyDetails(topoId, base);
            double sumOnHeap = 0.0;
            double sumOffHeap = 0.0;
            double sumCPU = 0.0;
            Assignment assignment = state.assignmentInfo(topoId, null);
            if (assignment != null) {
                if (assignment.is_set_worker_resources()) {
                    for (WorkerResources wr : assignment.get_worker_resources().values()) {
                        if (wr.is_set_cpu()) {
                            sumCPU += wr.get_cpu();
                        }
                        if (wr.is_set_mem_off_heap()) {
                            sumOffHeap += wr.get_mem_off_heap();
                        }
                        if (wr.is_set_mem_on_heap()) {
                            sumOnHeap += wr.get_mem_on_heap();
                        }
                    }
                }
            }
            ret = new TopologyResources(details.getTotalRequestedMemOnHeap(), details.getTotalRequestedMemOffHeap(), details.getTotalRequestedCpu(), sumOnHeap, sumOffHeap, sumCPU);
        } catch (KeyNotFoundException e) {
            //This can happen when a topology is first coming up
            // It's thrown by the blobstore code
            LOG.error("Failed to get topology details", e);
            ret = new TopologyResources(0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
        }
    }
    return ret;
}
Also used : Assignment(org.apache.storm.generated.Assignment) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) WorkerResources(org.apache.storm.generated.WorkerResources) IStormClusterState(org.apache.storm.cluster.IStormClusterState) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) KeyNotFoundException(org.apache.storm.generated.KeyNotFoundException)

Example 27 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class Nimbus method computeNewSchedulerAssignments.

private Map<String, SchedulerAssignment> computeNewSchedulerAssignments(Map<String, Assignment> existingAssignments, Topologies topologies, Map<String, StormBase> bases, String scratchTopologyId) throws KeyNotFoundException, AuthorizationException, InvalidTopologyException, IOException {
    Map<String, Set<List<Integer>>> topoToExec = computeTopologyToExecutors(bases);
    updateAllHeartbeats(existingAssignments, topoToExec);
    Map<String, Set<List<Integer>>> topoToAliveExecutors = computeTopologyToAliveExecutors(existingAssignments, topologies, topoToExec, scratchTopologyId);
    Map<String, Set<Long>> supervisorToDeadPorts = computeSupervisorToDeadPorts(existingAssignments, topoToExec, topoToAliveExecutors);
    Map<String, SchedulerAssignmentImpl> topoToSchedAssignment = computeTopologyToSchedulerAssignment(existingAssignments, topoToAliveExecutors);
    Set<String> missingAssignmentTopologies = new HashSet<>();
    for (TopologyDetails topo : topologies.getTopologies()) {
        String id = topo.getId();
        Set<List<Integer>> allExecs = topoToExec.get(id);
        Set<List<Integer>> aliveExecs = topoToAliveExecutors.get(id);
        int numDesiredWorkers = topo.getNumWorkers();
        int numAssignedWorkers = numUsedWorkers(topoToSchedAssignment.get(id));
        if (allExecs == null || allExecs.isEmpty() || !allExecs.equals(aliveExecs) || numDesiredWorkers > numAssignedWorkers) {
            //We have something to schedule...
            missingAssignmentTopologies.add(id);
        }
    }
    Map<String, SupervisorDetails> supervisors = readAllSupervisorDetails(supervisorToDeadPorts, topologies, missingAssignmentTopologies);
    Cluster cluster = new Cluster(inimbus, supervisors, topoToSchedAssignment, conf);
    cluster.setStatusMap(idToSchedStatus.get());
    scheduler.schedule(topologies, cluster);
    //merge with existing statuses
    idToSchedStatus.set(merge(idToSchedStatus.get(), cluster.getStatusMap()));
    nodeIdToResources.set(cluster.getSupervisorsResourcesMap());
    if (!Utils.getBoolean(conf.get(Config.SCHEDULER_DISPLAY_RESOURCE), false)) {
        cluster.updateAssignedMemoryForTopologyAndSupervisor(topologies);
    }
    // This is a hack for non-ras scheduler topology and worker resources
    Map<String, TopologyResources> resources = new HashMap<>();
    for (Entry<String, Double[]> uglyResources : cluster.getTopologyResourcesMap().entrySet()) {
        Double[] r = uglyResources.getValue();
        resources.put(uglyResources.getKey(), new TopologyResources(r[0], r[1], r[2], r[3], r[4], r[5]));
    }
    idToResources.getAndAccumulate(resources, (orig, update) -> merge(orig, update));
    Map<String, Map<WorkerSlot, WorkerResources>> workerResources = new HashMap<>();
    for (Entry<String, Map<WorkerSlot, Double[]>> uglyWorkerResources : cluster.getWorkerResourcesMap().entrySet()) {
        Map<WorkerSlot, WorkerResources> slotToResources = new HashMap<>();
        for (Entry<WorkerSlot, Double[]> uglySlotToResources : uglyWorkerResources.getValue().entrySet()) {
            Double[] r = uglySlotToResources.getValue();
            WorkerResources wr = new WorkerResources();
            wr.set_mem_on_heap(r[0]);
            wr.set_mem_off_heap(r[1]);
            wr.set_cpu(r[2]);
            slotToResources.put(uglySlotToResources.getKey(), wr);
        }
        workerResources.put(uglyWorkerResources.getKey(), slotToResources);
    }
    idToWorkerResources.getAndAccumulate(workerResources, (orig, update) -> merge(orig, update));
    return cluster.getAssignments();
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) SchedulerAssignmentImpl(org.apache.storm.scheduler.SchedulerAssignmentImpl) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) ArrayList(java.util.ArrayList) List(java.util.List) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) HashSet(java.util.HashSet) WorkerResources(org.apache.storm.generated.WorkerResources) Cluster(org.apache.storm.scheduler.Cluster) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) DataPoint(org.apache.storm.metric.api.DataPoint) Map(java.util.Map) TimeCacheMap(org.apache.storm.utils.TimeCacheMap) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 28 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class Nimbus method computeTopologyToAliveExecutors.

/**
     * compute a topology-id -> alive executors map
     * @param existingAssignment the current assignments
     * @param topologies the current topologies
     * @param topologyToExecutors the executors for the current topologies
     * @param scratchTopologyId the topology being rebalanced and should be excluded
     * @return the map of topology id to alive executors
     */
private Map<String, Set<List<Integer>>> computeTopologyToAliveExecutors(Map<String, Assignment> existingAssignment, Topologies topologies, Map<String, Set<List<Integer>>> topologyToExecutors, String scratchTopologyId) {
    Map<String, Set<List<Integer>>> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignment.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        TopologyDetails td = topologies.getById(topoId);
        Set<List<Integer>> allExecutors = topologyToExecutors.get(topoId);
        Set<List<Integer>> aliveExecutors;
        if (topoId.equals(scratchTopologyId)) {
            aliveExecutors = allExecutors;
        } else {
            aliveExecutors = new HashSet<>(aliveExecutors(td, allExecutors, assignment));
        }
        ret.put(topoId, aliveExecutors);
    }
    return ret;
}
Also used : Assignment(org.apache.storm.generated.Assignment) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) List(java.util.List) TopologyDetails(org.apache.storm.scheduler.TopologyDetails)

Example 29 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestDefaultEvictionStrategy method TestOverGuaranteeEviction.

/**
     * If users are above his or her guarantee, check if topology eviction works correct
     */
@Test
public void TestOverGuaranteeEviction() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 500);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 500);
    Map<String, Map<String, Number>> resourceUserPool = new HashMap<String, Map<String, Number>>();
    resourceUserPool.put("jerry", new HashMap<String, Number>());
    resourceUserPool.get("jerry").put("cpu", 70.0);
    resourceUserPool.get("jerry").put("memory", 700.0);
    resourceUserPool.put("bobby", new HashMap<String, Number>());
    resourceUserPool.get("bobby").put("cpu", 100.0);
    resourceUserPool.get("bobby").put("memory", 1000.0);
    resourceUserPool.put("derek", new HashMap<String, Number>());
    resourceUserPool.get("derek").put("cpu", 25.0);
    resourceUserPool.get("derek").put("memory", 250.0);
    config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS, resourceUserPool);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
    TopologyDetails topo1 = TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 20);
    TopologyDetails topo2 = TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
    TopologyDetails topo3 = TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 10);
    TopologyDetails topo4 = TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
    TopologyDetails topo5 = TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29);
    TopologyDetails topo6 = TestUtilsForResourceAwareScheduler.getTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 10);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo1.getId(), topo1);
    topoMap.put(topo3.getId(), topo3);
    topoMap.put(topo4.getId(), topo4);
    topoMap.put(topo5.getId(), topo5);
    Topologies topologies = new Topologies(topoMap);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    //user derek submits another topology into a full cluster
    //topo6 should not be able to scheduled intially, but since topo6 has higher priority than topo5
    //topo5 will be evicted so that topo6 can be scheduled
    topoMap.put(topo6.getId(), topo6);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 1, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    //topo5 will be evicted since topo6 has higher priority
    Assert.assertEquals("correct topology in attempted queue", "topo-5", rs.getUser("derek").getTopologiesAttempted().iterator().next().getName());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    //user jerry submits topo2
    topoMap.put(topo2.getId(), topo2);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 0, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 2, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    Assert.assertEquals("correct topology in attempted queue", "topo-6", rs.getUser("derek").getTopologiesAttempted().iterator().next().getName());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("Assert scheduling topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
}
Also used : HashMap(java.util.HashMap) Config(org.apache.storm.Config) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) SchedulerAssignmentImpl(org.apache.storm.scheduler.SchedulerAssignmentImpl) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.junit.Test)

Example 30 with TopologyDetails

use of org.apache.storm.scheduler.TopologyDetails in project storm by apache.

the class TestDefaultEvictionStrategy method TestEvictMultipleTopologies.

@Test
public void TestEvictMultipleTopologies() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 100.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1000.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
    Config config = new Config();
    config.putAll(Utils.readDefaultConfig());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    config.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    config.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    config.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 100.0);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 500);
    config.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 500);
    Map<String, Map<String, Number>> resourceUserPool = new HashMap<String, Map<String, Number>>();
    resourceUserPool.put("jerry", new HashMap<String, Number>());
    resourceUserPool.get("jerry").put("cpu", 200.0);
    resourceUserPool.get("jerry").put("memory", 2000.0);
    resourceUserPool.put("derek", new HashMap<String, Number>());
    resourceUserPool.get("derek").put("cpu", 100.0);
    resourceUserPool.get("derek").put("memory", 1000.0);
    config.put(Config.RESOURCE_AWARE_SCHEDULER_USER_POOLS, resourceUserPool);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "jerry");
    TopologyDetails topo1 = TestUtilsForResourceAwareScheduler.getTopology("topo-1", config, 2, 0, 1, 0, currentTime - 2, 10);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "bobby");
    TopologyDetails topo2 = TestUtilsForResourceAwareScheduler.getTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 10);
    TopologyDetails topo3 = TestUtilsForResourceAwareScheduler.getTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20);
    config.put(Config.TOPOLOGY_SUBMITTER_USER, "derek");
    TopologyDetails topo4 = TestUtilsForResourceAwareScheduler.getTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29);
    TopologyDetails topo5 = TestUtilsForResourceAwareScheduler.getTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo2.getId(), topo2);
    topoMap.put(topo3.getId(), topo3);
    topoMap.put(topo4.getId(), topo4);
    topoMap.put(topo5.getId(), topo5);
    Topologies topologies = new Topologies(topoMap);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(config);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("assert topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesRunning()) {
        Assert.assertTrue("assert topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("bobby").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("bobby").getTopologiesAttempted().size());
    //user jerry submits another topology
    topoMap.put(topo1.getId(), topo1);
    topologies = new Topologies(topoMap);
    rs.schedule(topologies, cluster);
    for (TopologyDetails topo : rs.getUser("jerry").getTopologiesRunning()) {
        Assert.assertTrue("assert topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 1, rs.getUser("jerry").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("jerry").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("jerry").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("jerry").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("derek").getTopologiesRunning()) {
        Assert.assertTrue("assert topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of running topologies", 2, rs.getUser("derek").getTopologiesRunning().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("derek").getTopologiesPending().size());
    Assert.assertEquals("# of attempted topologies", 0, rs.getUser("derek").getTopologiesAttempted().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("derek").getTopologiesInvalid().size());
    for (TopologyDetails topo : rs.getUser("bobby").getTopologiesAttempted()) {
        Assert.assertFalse("assert topology success", TestUtilsForResourceAwareScheduler.assertStatusSuccess(cluster.getStatusMap().get(topo.getId())));
    }
    Assert.assertEquals("# of attempted topologies", 2, rs.getUser("bobby").getTopologiesAttempted().size());
    Assert.assertEquals("# of pending topologies", 0, rs.getUser("bobby").getTopologiesPending().size());
    Assert.assertEquals("# of invalid topologies", 0, rs.getUser("bobby").getTopologiesInvalid().size());
    Assert.assertEquals("# of running topologies", 0, rs.getUser("bobby").getTopologiesRunning().size());
}
Also used : HashMap(java.util.HashMap) Config(org.apache.storm.Config) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) SchedulerAssignmentImpl(org.apache.storm.scheduler.SchedulerAssignmentImpl) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.junit.Test)

Aggregations

TopologyDetails (org.apache.storm.scheduler.TopologyDetails)47 HashMap (java.util.HashMap)35 SupervisorDetails (org.apache.storm.scheduler.SupervisorDetails)27 Config (org.apache.storm.Config)26 Topologies (org.apache.storm.scheduler.Topologies)26 Test (org.junit.Test)26 Cluster (org.apache.storm.scheduler.Cluster)24 SchedulerAssignmentImpl (org.apache.storm.scheduler.SchedulerAssignmentImpl)24 INimbus (org.apache.storm.scheduler.INimbus)23 Map (java.util.Map)22 ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)18 WorkerSlot (org.apache.storm.scheduler.WorkerSlot)15 ArrayList (java.util.ArrayList)13 SchedulerAssignment (org.apache.storm.scheduler.SchedulerAssignment)12 List (java.util.List)10 StormTopology (org.apache.storm.generated.StormTopology)10 HashSet (java.util.HashSet)8 LinkedList (java.util.LinkedList)8 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)8 ResourceAwareScheduler (org.apache.storm.scheduler.resource.ResourceAwareScheduler)6