Examples with ResourceMetrics org.apache.storm.scheduler.resource.normalization.ResourceMetrics used on opensource projects

Example 1 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testHeterogeneousCluster.

public void testHeterogeneousCluster(Config topologyConf, String strategyName) {
    LOG.info("\n\n\t\ttestHeterogeneousCluster");
    INimbus iNimbus = new INimbusTest();
    // strong supervisor node
    Map<String, Double> resourceMap1 = new HashMap<>();
    resourceMap1.put(Config.SUPERVISOR_CPU_CAPACITY, 800.0);
    resourceMap1.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 4096.0);
    // weak supervisor node
    Map<String, Double> resourceMap2 = new HashMap<>();
    resourceMap2.put(Config.SUPERVISOR_CPU_CAPACITY, 200.0);
    resourceMap2.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0);
    resourceMap1 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap1);
    resourceMap2 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap2);
    Map<String, SupervisorDetails> supMap = new HashMap<>();
    for (int i = 0; i < 2; i++) {
        List<Number> ports = new LinkedList<>();
        for (int j = 0; j < 4; j++) {
            ports.add(j);
        }
        SupervisorDetails sup = new SupervisorDetails("r00s00" + i, "host-" + i, null, ports, i == 0 ? resourceMap1 : resourceMap2);
        supMap.put(sup.getId(), sup);
    }
    LOG.info("SUPERVISORS = {}", supMap);
    // topo1 has one single huge task that can not be handled by the small-super
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0).setMemoryLoad(2000.0, 48.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0, "user");
    // topo2 has 4 large tasks
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0).setMemoryLoad(500.0, 12.0);
    StormTopology stormTopology2 = builder2.createTopology();
    Config config2 = new Config();
    config2.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0, "user");
    // topo3 has 4 large tasks
    TopologyBuilder builder3 = new TopologyBuilder();
    builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0).setMemoryLoad(200.0, 56.0);
    StormTopology stormTopology3 = builder3.createTopology();
    Config config3 = new Config();
    config3.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap3 = genExecsAndComps(stormTopology3);
    TopologyDetails topology3 = new TopologyDetails("topology3", config3, stormTopology3, 1, executorMap3, 0, "user");
    // topo4 has 12 small tasks, whose mem usage does not exactly divide a node's mem capacity
    TopologyBuilder builder4 = new TopologyBuilder();
    builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0).setMemoryLoad(100.0, 0.0);
    StormTopology stormTopology4 = builder4.createTopology();
    Config config4 = new Config();
    config4.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap4 = genExecsAndComps(stormTopology4);
    TopologyDetails topology4 = new TopologyDetails("topology4", config4, stormTopology4, 1, executorMap4, 0, "user");
    // topo5 has 40 small tasks, it should be able to exactly use up both the cpu and mem in the cluster
    TopologyBuilder builder5 = new TopologyBuilder();
    builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0).setMemoryLoad(100.0, 28.0);
    StormTopology stormTopology5 = builder5.createTopology();
    Config config5 = new Config();
    config5.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap5 = genExecsAndComps(stormTopology5);
    TopologyDetails topology5 = new TopologyDetails("topology5", config5, stormTopology5, 1, executorMap5, 0, "user");
    // Test1: Launch topo 1-3 together, it should be able to use up either mem or cpu resource due to exact division
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    LOG.info("\n\n\t\tScheduling topologies 1, 2 and 3");
    Topologies topologies = new Topologies(topology1, topology2, topology3);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    Map<SupervisorDetails, Double> superToCpu = null;
    Map<SupervisorDetails, Double> superToMem = null;
    try {
        rs.schedule(topologies, cluster);
        assertFalse(cluster.needsSchedulingRas(topology1));
        assertFalse(cluster.needsSchedulingRas(topology2));
        assertFalse(cluster.needsSchedulingRas(topology3));
        String expectedMsgPrefix = "Running - Fully Scheduled by " + strategyName;
        assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith(expectedMsgPrefix));
        assertTrue(cluster.getStatusMap().get(topology2.getId()).startsWith(expectedMsgPrefix));
        assertTrue(cluster.getStatusMap().get(topology3.getId()).startsWith(expectedMsgPrefix));
        superToCpu = getSupervisorToCpuUsage(cluster, topologies);
        superToMem = getSupervisorToMemoryUsage(cluster, topologies);
        final Double EPSILON = 0.0001;
        for (SupervisorDetails supervisor : supMap.values()) {
            Double cpuAvailable = supervisor.getTotalCpu();
            Double memAvailable = supervisor.getTotalMemory();
            Double cpuUsed = superToCpu.get(supervisor);
            Double memUsed = superToMem.get(supervisor);
            assertTrue(supervisor.getId() + " MEM: " + memAvailable + " == " + memUsed + " OR CPU: " + cpuAvailable + " == " + cpuUsed, (Math.abs(memAvailable - memUsed) < EPSILON) || (Math.abs(cpuAvailable - cpuUsed) < EPSILON));
        }
    } finally {
        rs.cleanup();
    }
    // end of Test1
    LOG.warn("\n\n\t\tSwitching to topologies 1, 2 and 4");
    // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so one of the 3 topos will not be
    // scheduled
    topologies = new Topologies(topology1, topology2, topology4);
    cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        int numTopologiesAssigned = 0;
        if (cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 1 scheduled");
            numTopologiesAssigned++;
        }
        if (cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 2 scheduled");
            numTopologiesAssigned++;
        }
        if (cluster.getStatusMap().get(topology4.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 3 scheduled");
            numTopologiesAssigned++;
        }
        assertEquals(2, numTopologiesAssigned);
    } finally {
        rs.cleanup();
    }
    // end of Test2
    LOG.info("\n\n\t\tScheduling just topo 5");
    // Test3: "Launch topo5 only, both mem and cpu should be exactly used up"
    topologies = new Topologies(topology5);
    cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        superToCpu = getSupervisorToCpuUsage(cluster, topologies);
        superToMem = getSupervisorToMemoryUsage(cluster, topologies);
        for (SupervisorDetails supervisor : supMap.values()) {
            Double cpuAvailable = supervisor.getTotalCpu();
            Double memAvailable = supervisor.getTotalMemory();
            Double cpuUsed = superToCpu.get(supervisor);
            Double memUsed = superToMem.get(supervisor);
            assertEquals(cpuAvailable, cpuUsed, 0.0001);
            assertEquals(memAvailable, memUsed, 0.0001);
        }
    } finally {
        rs.cleanup();
    }
// end of Test3
}

Example 2 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method minCpuWorkerSplitFails.

/**
 * Min CPU for worker set to 50%.  1 supervisor with 100% CPU.
 * A topology with 3 workers should fail scheduling even if under CPU.
 */
@Test
public void minCpuWorkerSplitFails() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(1, 4, 100, 60000);
    Config config = createClusterConfig(10, 500, 500, null);
    config.put(DaemonConfig.STORM_WORKER_MIN_CPU_PCORE_PERCENT, 50.0);
    TopologyDetails topo1 = genTopology("topo-1", config, 10, 0, 1, 1, currentTime - 2, 20, "jerry", 2000.0);
    Topologies topologies = new Topologies(topo1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertTrue(cluster.needsSchedulingRas(topo1));
    assertFalse("Topo-1 unscheduled?", cluster.getAssignmentById(topo1.getId()) != null);
}

Example 3 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testLargeClusterSchedulingTiming.

/*
     * Test time to schedule large cluster scheduling with fragmentation
     */
private TimeBlockResult testLargeClusterSchedulingTiming(int numNodes, Config config) {
    // Attempt to schedule multiple copies of 2 different topologies (topo-t0 and topo-t1) in 3 blocks.
    // Without fragmentation it is possible to schedule all topologies, but fragmentation causes topologies to not
    // schedule for the last block.
    // Get start/end indices for blocks
    int numTopologyPairs = numNodes;
    int increment = (int) Math.floor(numTopologyPairs * 0.1);
    int[] firstBlockIndices = { 0, increment - 1 };
    int[] midBlockIndices = { increment, numTopologyPairs - increment - 1 };
    int[] lastBlockIndices = { numTopologyPairs - increment, numTopologyPairs - 1 };
    // Memory is the constraining resource.
    // memory required by topo-t0
    double t0Mem = 70;
    // memory required by topo-t1
    double t1Mem = 20;
    double nodeMem = 100;
    // first block (0% - 10%)
    Map<String, TopologyDetails> topologyMap = new HashMap<>();
    addTopologyBlockToMap(topologyMap, "topo_t0-", config, t0Mem, firstBlockIndices);
    addTopologyBlockToMap(topologyMap, "topo_t1-", config, t1Mem, firstBlockIndices);
    Topologies topologies = new Topologies(topologyMap);
    Map<String, SupervisorDetails> supMap = genSupervisors(numNodes, 7, 3500, nodeMem);
    Cluster cluster = new Cluster(new INimbusTest(), new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<String, SchedulerAssignmentImpl>(), topologies, config);
    TimeBlockResult timeBlockResult = new TimeBlockResult();
    // schedule first block (0% - 10%)
    {
        scheduler = new ResourceAwareScheduler();
        scheduler.prepare(config, new StormMetricsRegistry());
        long time = Time.currentTimeMillis();
        scheduler.schedule(topologies, cluster);
        timeBlockResult.firstBlockTime.add(Time.currentTimeMillis() - time);
    }
    // schedule mid block (10% - 90%)
    {
        addTopologyBlockToMap(topologyMap, "topo_t0-", config, t0Mem, midBlockIndices);
        addTopologyBlockToMap(topologyMap, "topo_t1-", config, t1Mem, midBlockIndices);
        topologies = new Topologies(topologyMap);
        cluster = new Cluster(cluster, topologies);
        scheduler.schedule(topologies, cluster);
    }
    // schedule last block (90% to 100%)
    {
        addTopologyBlockToMap(topologyMap, "topo_t0-", config, t0Mem, lastBlockIndices);
        addTopologyBlockToMap(topologyMap, "topo_t1-", config, t1Mem, lastBlockIndices);
        topologies = new Topologies(topologyMap);
        cluster = new Cluster(cluster, topologies);
        long time = Time.currentTimeMillis();
        scheduler.schedule(topologies, cluster);
        timeBlockResult.lastBlockTime.add(Time.currentTimeMillis() - time);
    }
    return timeBlockResult;
}

Example 4 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testSubmitUsersWithNoGuarantees.

@Test
public void testSubmitUsersWithNoGuarantees() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 200, 2000));
    Config config = createClusterConfig(100, 500, 500, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 10, "jerry"), genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-2", "topo-3", "topo-4");
    assertTopologiesNotScheduled(cluster, "topo-5");
}

Example 5 with ResourceMetrics

use of org.apache.storm.scheduler.resource.normalization.ResourceMetrics in project storm by apache.

the class TestResourceAwareScheduler method testResourceLimitation.

@Test
public void testResourceLimitation() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0).setMemoryLoad(1000.0, 200.0);
    builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0).setMemoryLoad(500.0, 100.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 2, executorMap1, 0, "user");
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    scheduler = rs;
    Topologies topologies = new Topologies(topology1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    rs.prepare(config, new StormMetricsRegistry());
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Set<WorkerSlot> assignedSlots1 = assignment1.getSlots();
    Set<String> nodesIDs1 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots1) {
        nodesIDs1.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    List<Double> assignedExecutorMemory = new ArrayList<>();
    List<Double> assignedExecutorCpu = new ArrayList<>();
    for (ExecutorDetails executor : executors1) {
        assignedExecutorMemory.add(topology1.getTotalMemReqTask(executor));
        assignedExecutorCpu.add(topology1.getTotalCpuReqTask(executor));
    }
    Collections.sort(assignedExecutorCpu);
    Collections.sort(assignedExecutorMemory);
    Map<ExecutorDetails, SupervisorDetails> executorToSupervisor = new HashMap<>();
    Map<SupervisorDetails, List<ExecutorDetails>> supervisorToExecutors = new HashMap<>();
    Map<Double, Double> cpuAvailableToUsed = new HashMap<>();
    Map<Double, Double> memoryAvailableToUsed = new HashMap<>();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : assignment1.getExecutorToSlot().entrySet()) {
        executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId()));
    }
    for (Map.Entry<ExecutorDetails, SupervisorDetails> entry : executorToSupervisor.entrySet()) {
        supervisorToExecutors.computeIfAbsent(entry.getValue(), k -> new ArrayList<>()).add(entry.getKey());
    }
    for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
        Double supervisorTotalCpu = entry.getKey().getTotalCpu();
        Double supervisorTotalMemory = entry.getKey().getTotalMemory();
        Double supervisorUsedCpu = 0.0;
        Double supervisorUsedMemory = 0.0;
        for (ExecutorDetails executor : entry.getValue()) {
            supervisorUsedMemory += topology1.getTotalCpuReqTask(executor);
            supervisorTotalCpu += topology1.getTotalMemReqTask(executor);
        }
        cpuAvailableToUsed.put(supervisorTotalCpu, supervisorUsedCpu);
        memoryAvailableToUsed.put(supervisorTotalMemory, supervisorUsedMemory);
    }
    // executor0 resides one one worker (on one), executor1 and executor2 on another worker (on the other node)
    assertEquals(2, assignedSlots1.size());
    assertEquals(2, nodesIDs1.size());
    assertEquals(3, executors1.size());
    assertEquals(100.0, assignedExecutorCpu.get(0), 0.001);
    assertEquals(250.0, assignedExecutorCpu.get(1), 0.001);
    assertEquals(250.0, assignedExecutorCpu.get(2), 0.001);
    assertEquals(600.0, assignedExecutorMemory.get(0), 0.001);
    assertEquals(1200.0, assignedExecutorMemory.get(1), 0.001);
    assertEquals(1200.0, assignedExecutorMemory.get(2), 0.001);
    for (Map.Entry<Double, Double> entry : memoryAvailableToUsed.entrySet()) {
        assertTrue(entry.getKey() - entry.getValue() >= 0);
    }
    for (Map.Entry<Double, Double> entry : cpuAvailableToUsed.entrySet()) {
        assertTrue(entry.getKey() - entry.getValue() >= 0);
    }
    assertFalse(cluster.needsSchedulingRas(topology1));
    assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy"));
}