Examples with ExecutorDetails org.apache.storm.scheduler.ExecutorDetails used on opensource projects

Example 21 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class Nimbus method computeTopologyToSchedulerAssignment.

/**
 * Convert assignment information in zk to SchedulerAssignment, so it can be used by scheduler api.
 *
 * @param existingAssignments      current assignments
 * @param topologyToAliveExecutors executors that are alive
 * @return topo ID to schedulerAssignment
 */
private Map<String, SchedulerAssignmentImpl> computeTopologyToSchedulerAssignment(Map<String, Assignment> existingAssignments, Map<String, Set<List<Integer>>> topologyToAliveExecutors) {
    Map<String, SchedulerAssignmentImpl> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignments.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        Set<List<Integer>> aliveExecutors = topologyToAliveExecutors.get(topoId);
        Map<List<Long>, NodeInfo> execToNodePort = assignment.get_executor_node_port();
        Map<NodeInfo, WorkerResources> workerToResources = assignment.get_worker_resources();
        Map<NodeInfo, WorkerSlot> nodePortToSlot = new HashMap<>();
        Map<WorkerSlot, WorkerResources> slotToResources = new HashMap<>();
        for (Entry<NodeInfo, WorkerResources> nodeAndResources : workerToResources.entrySet()) {
            NodeInfo info = nodeAndResources.getKey();
            WorkerResources resources = nodeAndResources.getValue();
            WorkerSlot slot = new WorkerSlot(info.get_node(), info.get_port_iterator().next());
            nodePortToSlot.put(info, slot);
            slotToResources.put(slot, resources);
        }
        Map<ExecutorDetails, WorkerSlot> execToSlot = new HashMap<>();
        for (Entry<List<Long>, NodeInfo> execAndNodePort : execToNodePort.entrySet()) {
            List<Integer> exec = asIntExec(execAndNodePort.getKey());
            NodeInfo info = execAndNodePort.getValue();
            if (aliveExecutors.contains(exec)) {
                execToSlot.put(new ExecutorDetails(exec.get(0), exec.get(1)), nodePortToSlot.get(info));
            }
        }
        ret.put(topoId, new SchedulerAssignmentImpl(topoId, execToSlot, slotToResources, null));
    }
    return ret;
}

Example 22 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class Nimbus method readTopologyDetails.

private TopologyDetails readTopologyDetails(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
    assert (base != null);
    assert (topoId != null);
    Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, topoCache);
    StormTopology topo = readStormTopologyAsNimbus(topoId, topoCache);
    if (!base.is_set_principal()) {
        fixupBase(base, topoConf);
        stormClusterState.updateStorm(topoId, base);
    }
    Map<List<Integer>, String> rawExecToComponent = computeExecutorToComponent(topoId, base, topoConf, topo);
    Map<ExecutorDetails, String> executorsToComponent = new HashMap<>();
    for (Entry<List<Integer>, String> entry : rawExecToComponent.entrySet()) {
        List<Integer> execs = entry.getKey();
        ExecutorDetails execDetails = new ExecutorDetails(execs.get(0), execs.get(1));
        executorsToComponent.put(execDetails, entry.getValue());
    }
    return new TopologyDetails(topoId, topoConf, topo, base.get_num_workers(), executorsToComponent, base.get_launch_time_secs(), base.get_owner());
}

Example 23 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestResourceAwareScheduler method testHeterogeneousCluster.

public void testHeterogeneousCluster(Config topologyConf, String strategyName) {
    LOG.info("\n\n\t\ttestHeterogeneousCluster");
    INimbus iNimbus = new INimbusTest();
    // strong supervisor node
    Map<String, Double> resourceMap1 = new HashMap<>();
    resourceMap1.put(Config.SUPERVISOR_CPU_CAPACITY, 800.0);
    resourceMap1.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 4096.0);
    // weak supervisor node
    Map<String, Double> resourceMap2 = new HashMap<>();
    resourceMap2.put(Config.SUPERVISOR_CPU_CAPACITY, 200.0);
    resourceMap2.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0);
    resourceMap1 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap1);
    resourceMap2 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap2);
    Map<String, SupervisorDetails> supMap = new HashMap<>();
    for (int i = 0; i < 2; i++) {
        List<Number> ports = new LinkedList<>();
        for (int j = 0; j < 4; j++) {
            ports.add(j);
        }
        SupervisorDetails sup = new SupervisorDetails("r00s00" + i, "host-" + i, null, ports, i == 0 ? resourceMap1 : resourceMap2);
        supMap.put(sup.getId(), sup);
    }
    LOG.info("SUPERVISORS = {}", supMap);
    // topo1 has one single huge task that can not be handled by the small-super
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0).setMemoryLoad(2000.0, 48.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0, "user");
    // topo2 has 4 large tasks
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0).setMemoryLoad(500.0, 12.0);
    StormTopology stormTopology2 = builder2.createTopology();
    Config config2 = new Config();
    config2.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0, "user");
    // topo3 has 4 large tasks
    TopologyBuilder builder3 = new TopologyBuilder();
    builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0).setMemoryLoad(200.0, 56.0);
    StormTopology stormTopology3 = builder3.createTopology();
    Config config3 = new Config();
    config3.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap3 = genExecsAndComps(stormTopology3);
    TopologyDetails topology3 = new TopologyDetails("topology3", config3, stormTopology3, 1, executorMap3, 0, "user");
    // topo4 has 12 small tasks, whose mem usage does not exactly divide a node's mem capacity
    TopologyBuilder builder4 = new TopologyBuilder();
    builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0).setMemoryLoad(100.0, 0.0);
    StormTopology stormTopology4 = builder4.createTopology();
    Config config4 = new Config();
    config4.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap4 = genExecsAndComps(stormTopology4);
    TopologyDetails topology4 = new TopologyDetails("topology4", config4, stormTopology4, 1, executorMap4, 0, "user");
    // topo5 has 40 small tasks, it should be able to exactly use up both the cpu and mem in the cluster
    TopologyBuilder builder5 = new TopologyBuilder();
    builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0).setMemoryLoad(100.0, 28.0);
    StormTopology stormTopology5 = builder5.createTopology();
    Config config5 = new Config();
    config5.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap5 = genExecsAndComps(stormTopology5);
    TopologyDetails topology5 = new TopologyDetails("topology5", config5, stormTopology5, 1, executorMap5, 0, "user");
    // Test1: Launch topo 1-3 together, it should be able to use up either mem or cpu resource due to exact division
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    LOG.info("\n\n\t\tScheduling topologies 1, 2 and 3");
    Topologies topologies = new Topologies(topology1, topology2, topology3);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    Map<SupervisorDetails, Double> superToCpu = null;
    Map<SupervisorDetails, Double> superToMem = null;
    try {
        rs.schedule(topologies, cluster);
        assertFalse(cluster.needsSchedulingRas(topology1));
        assertFalse(cluster.needsSchedulingRas(topology2));
        assertFalse(cluster.needsSchedulingRas(topology3));
        String expectedMsgPrefix = "Running - Fully Scheduled by " + strategyName;
        assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith(expectedMsgPrefix));
        assertTrue(cluster.getStatusMap().get(topology2.getId()).startsWith(expectedMsgPrefix));
        assertTrue(cluster.getStatusMap().get(topology3.getId()).startsWith(expectedMsgPrefix));
        superToCpu = getSupervisorToCpuUsage(cluster, topologies);
        superToMem = getSupervisorToMemoryUsage(cluster, topologies);
        final Double EPSILON = 0.0001;
        for (SupervisorDetails supervisor : supMap.values()) {
            Double cpuAvailable = supervisor.getTotalCpu();
            Double memAvailable = supervisor.getTotalMemory();
            Double cpuUsed = superToCpu.get(supervisor);
            Double memUsed = superToMem.get(supervisor);
            assertTrue(supervisor.getId() + " MEM: " + memAvailable + " == " + memUsed + " OR CPU: " + cpuAvailable + " == " + cpuUsed, (Math.abs(memAvailable - memUsed) < EPSILON) || (Math.abs(cpuAvailable - cpuUsed) < EPSILON));
        }
    } finally {
        rs.cleanup();
    }
    // end of Test1
    LOG.warn("\n\n\t\tSwitching to topologies 1, 2 and 4");
    // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so one of the 3 topos will not be
    // scheduled
    topologies = new Topologies(topology1, topology2, topology4);
    cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        int numTopologiesAssigned = 0;
        if (cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 1 scheduled");
            numTopologiesAssigned++;
        }
        if (cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 2 scheduled");
            numTopologiesAssigned++;
        }
        if (cluster.getStatusMap().get(topology4.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 3 scheduled");
            numTopologiesAssigned++;
        }
        assertEquals(2, numTopologiesAssigned);
    } finally {
        rs.cleanup();
    }
    // end of Test2
    LOG.info("\n\n\t\tScheduling just topo 5");
    // Test3: "Launch topo5 only, both mem and cpu should be exactly used up"
    topologies = new Topologies(topology5);
    cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        superToCpu = getSupervisorToCpuUsage(cluster, topologies);
        superToMem = getSupervisorToMemoryUsage(cluster, topologies);
        for (SupervisorDetails supervisor : supMap.values()) {
            Double cpuAvailable = supervisor.getTotalCpu();
            Double memAvailable = supervisor.getTotalMemory();
            Double cpuUsed = superToCpu.get(supervisor);
            Double memUsed = superToMem.get(supervisor);
            assertEquals(cpuAvailable, cpuUsed, 0.0001);
            assertEquals(memAvailable, memUsed, 0.0001);
        }
    } finally {
        rs.cleanup();
    }
// end of Test3
}

Example 24 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestResourceAwareScheduler method addTopologyBlockToMap.

// Create multiple copies of a test topology
private void addTopologyBlockToMap(Map<String, TopologyDetails> topologyMap, String baseName, Config config, double spoutMemoryLoad, int[] blockIndices) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("testSpout", new TestSpout(), 1).setMemoryLoad(spoutMemoryLoad);
    StormTopology stormTopology = builder.createTopology();
    Map<ExecutorDetails, String> executorMap = genExecsAndComps(stormTopology);
    for (int i = blockIndices[0]; i <= blockIndices[1]; ++i) {
        TopologyDetails topo = new TopologyDetails(baseName + i, config, stormTopology, 0, executorMap, 0, "user");
        topologyMap.put(topo.getId(), topo);
    }
}

Example 25 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestResourceAwareScheduler method testResourceLimitation.

@Test
public void testResourceLimitation() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout", new TestWordSpout(), 2).setCPULoad(250.0).setMemoryLoad(1000.0, 200.0);
    builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(100.0).setMemoryLoad(500.0, 100.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 2, executorMap1, 0, "user");
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    scheduler = rs;
    Topologies topologies = new Topologies(topology1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    rs.prepare(config, new StormMetricsRegistry());
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Set<WorkerSlot> assignedSlots1 = assignment1.getSlots();
    Set<String> nodesIDs1 = new HashSet<>();
    for (WorkerSlot slot : assignedSlots1) {
        nodesIDs1.add(slot.getNodeId());
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    List<Double> assignedExecutorMemory = new ArrayList<>();
    List<Double> assignedExecutorCpu = new ArrayList<>();
    for (ExecutorDetails executor : executors1) {
        assignedExecutorMemory.add(topology1.getTotalMemReqTask(executor));
        assignedExecutorCpu.add(topology1.getTotalCpuReqTask(executor));
    }
    Collections.sort(assignedExecutorCpu);
    Collections.sort(assignedExecutorMemory);
    Map<ExecutorDetails, SupervisorDetails> executorToSupervisor = new HashMap<>();
    Map<SupervisorDetails, List<ExecutorDetails>> supervisorToExecutors = new HashMap<>();
    Map<Double, Double> cpuAvailableToUsed = new HashMap<>();
    Map<Double, Double> memoryAvailableToUsed = new HashMap<>();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : assignment1.getExecutorToSlot().entrySet()) {
        executorToSupervisor.put(entry.getKey(), cluster.getSupervisorById(entry.getValue().getNodeId()));
    }
    for (Map.Entry<ExecutorDetails, SupervisorDetails> entry : executorToSupervisor.entrySet()) {
        supervisorToExecutors.computeIfAbsent(entry.getValue(), k -> new ArrayList<>()).add(entry.getKey());
    }
    for (Map.Entry<SupervisorDetails, List<ExecutorDetails>> entry : supervisorToExecutors.entrySet()) {
        Double supervisorTotalCpu = entry.getKey().getTotalCpu();
        Double supervisorTotalMemory = entry.getKey().getTotalMemory();
        Double supervisorUsedCpu = 0.0;
        Double supervisorUsedMemory = 0.0;
        for (ExecutorDetails executor : entry.getValue()) {
            supervisorUsedMemory += topology1.getTotalCpuReqTask(executor);
            supervisorTotalCpu += topology1.getTotalMemReqTask(executor);
        }
        cpuAvailableToUsed.put(supervisorTotalCpu, supervisorUsedCpu);
        memoryAvailableToUsed.put(supervisorTotalMemory, supervisorUsedMemory);
    }
    // executor0 resides one one worker (on one), executor1 and executor2 on another worker (on the other node)
    assertEquals(2, assignedSlots1.size());
    assertEquals(2, nodesIDs1.size());
    assertEquals(3, executors1.size());
    assertEquals(100.0, assignedExecutorCpu.get(0), 0.001);
    assertEquals(250.0, assignedExecutorCpu.get(1), 0.001);
    assertEquals(250.0, assignedExecutorCpu.get(2), 0.001);
    assertEquals(600.0, assignedExecutorMemory.get(0), 0.001);
    assertEquals(1200.0, assignedExecutorMemory.get(1), 0.001);
    assertEquals(1200.0, assignedExecutorMemory.get(2), 0.001);
    for (Map.Entry<Double, Double> entry : memoryAvailableToUsed.entrySet()) {
        assertTrue(entry.getKey() - entry.getValue() >= 0);
    }
    for (Map.Entry<Double, Double> entry : cpuAvailableToUsed.entrySet()) {
        assertTrue(entry.getKey() - entry.getValue() >= 0);
    }
    assertFalse(cluster.needsSchedulingRas(topology1));
    assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy"));
}