Examples with ExecutorDetails org.apache.storm.scheduler.ExecutorDetails used on opensource projects

Example 46 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class RAS_Node method getCpuUsedByWorker.

/**
     * get the amount of cpu used by a worker
     */
public double getCpuUsedByWorker(WorkerSlot ws) {
    TopologyDetails topo = findTopologyUsingWorker(ws);
    if (topo == null) {
        return 0.0;
    }
    Collection<ExecutorDetails> execs = getExecutors(ws, _cluster);
    double totalCpuUsed = 0.0;
    for (ExecutorDetails exec : execs) {
        totalCpuUsed += topo.getTotalCpuReqTask(exec);
    }
    return totalCpuUsed;
}

Example 47 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class RAS_Nodes method getAllNodesFrom.

public static Map<String, RAS_Node> getAllNodesFrom(Cluster cluster, Topologies topologies) {
    //A map of node ids to node objects
    Map<String, RAS_Node> nodeIdToNode = new HashMap<String, RAS_Node>();
    //A map of assignments organized by node with the following format:
    //{nodeId -> {topologyId -> {workerId -> {execs}}}}
    Map<String, Map<String, Map<String, Collection<ExecutorDetails>>>> assignmentRelationshipMap = new HashMap<String, Map<String, Map<String, Collection<ExecutorDetails>>>>();
    Map<String, Map<String, WorkerSlot>> workerIdToWorker = new HashMap<String, Map<String, WorkerSlot>>();
    for (SchedulerAssignment assignment : cluster.getAssignments().values()) {
        String topId = assignment.getTopologyId();
        for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> entry : assignment.getSlotToExecutors().entrySet()) {
            WorkerSlot slot = entry.getKey();
            String nodeId = slot.getNodeId();
            Collection<ExecutorDetails> execs = entry.getValue();
            if (!assignmentRelationshipMap.containsKey(nodeId)) {
                assignmentRelationshipMap.put(nodeId, new HashMap<String, Map<String, Collection<ExecutorDetails>>>());
                workerIdToWorker.put(nodeId, new HashMap<String, WorkerSlot>());
            }
            workerIdToWorker.get(nodeId).put(slot.getId(), slot);
            if (!assignmentRelationshipMap.get(nodeId).containsKey(topId)) {
                assignmentRelationshipMap.get(nodeId).put(topId, new HashMap<String, Collection<ExecutorDetails>>());
            }
            if (!assignmentRelationshipMap.get(nodeId).get(topId).containsKey(slot.getId())) {
                assignmentRelationshipMap.get(nodeId).get(topId).put(slot.getId(), new LinkedList<ExecutorDetails>());
            }
            assignmentRelationshipMap.get(nodeId).get(topId).get(slot.getId()).addAll(execs);
        }
    }
    for (SupervisorDetails sup : cluster.getSupervisors().values()) {
        //Initialize a worker slot for every port even if there is no assignment to it
        for (int port : sup.getAllPorts()) {
            WorkerSlot worker = new WorkerSlot(sup.getId(), port);
            if (!workerIdToWorker.containsKey(sup.getId())) {
                workerIdToWorker.put(sup.getId(), new HashMap<String, WorkerSlot>());
            }
            if (!workerIdToWorker.get(sup.getId()).containsKey(worker.getId())) {
                workerIdToWorker.get(sup.getId()).put(worker.getId(), worker);
            }
        }
        nodeIdToNode.put(sup.getId(), new RAS_Node(sup.getId(), sup, cluster, topologies, workerIdToWorker.get(sup.getId()), assignmentRelationshipMap.get(sup.getId())));
    }
    //Add in supervisors that might have crashed but workers are still alive
    for (Map.Entry<String, Map<String, Map<String, Collection<ExecutorDetails>>>> entry : assignmentRelationshipMap.entrySet()) {
        String nodeId = entry.getKey();
        Map<String, Map<String, Collection<ExecutorDetails>>> assignments = entry.getValue();
        if (!nodeIdToNode.containsKey(nodeId)) {
            LOG.info("Found an assigned slot(s) on a dead supervisor {} with assignments {}", nodeId, assignments);
            nodeIdToNode.put(nodeId, new RAS_Node(nodeId, null, cluster, topologies, workerIdToWorker.get(nodeId), assignments));
        }
    }
    return nodeIdToNode;
}

Example 48 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class ResourceAwareScheduler method mkAssignment.

private boolean mkAssignment(TopologyDetails td, Map<WorkerSlot, Collection<ExecutorDetails>> schedulerAssignmentMap) {
    if (schedulerAssignmentMap != null) {
        double requestedMemOnHeap = td.getTotalRequestedMemOnHeap();
        double requestedMemOffHeap = td.getTotalRequestedMemOffHeap();
        double requestedCpu = td.getTotalRequestedCpu();
        double assignedMemOnHeap = 0.0;
        double assignedMemOffHeap = 0.0;
        double assignedCpu = 0.0;
        Map<WorkerSlot, Double[]> workerResources = new HashMap<WorkerSlot, Double[]>();
        Set<String> nodesUsed = new HashSet<String>();
        for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> workerToTasksEntry : schedulerAssignmentMap.entrySet()) {
            WorkerSlot targetSlot = workerToTasksEntry.getKey();
            Collection<ExecutorDetails> execsNeedScheduling = workerToTasksEntry.getValue();
            RAS_Node targetNode = this.schedulingState.nodes.getNodeById(targetSlot.getNodeId());
            targetSlot = allocateResourceToSlot(td, execsNeedScheduling, targetSlot);
            targetNode.assign(targetSlot, td, execsNeedScheduling);
            LOG.debug("ASSIGNMENT    TOPOLOGY: {}  TASKS: {} To Node: {} on Slot: {}", td.getName(), execsNeedScheduling, targetNode.getHostname(), targetSlot.getPort());
            for (ExecutorDetails exec : execsNeedScheduling) {
                targetNode.consumeResourcesforTask(exec, td);
            }
            if (!nodesUsed.contains(targetNode.getId())) {
                nodesUsed.add(targetNode.getId());
            }
            assignedMemOnHeap += targetSlot.getAllocatedMemOnHeap();
            assignedMemOffHeap += targetSlot.getAllocatedMemOffHeap();
            assignedCpu += targetSlot.getAllocatedCpu();
            Double[] worker_resources = { requestedMemOnHeap, requestedMemOffHeap, requestedCpu, targetSlot.getAllocatedMemOnHeap(), targetSlot.getAllocatedMemOffHeap(), targetSlot.getAllocatedCpu() };
            workerResources.put(targetSlot, worker_resources);
        }
        Double[] resources = { requestedMemOnHeap, requestedMemOffHeap, requestedCpu, assignedMemOnHeap, assignedMemOffHeap, assignedCpu };
        LOG.debug("setTopologyResources for {}: requested on-heap mem, off-heap mem, cpu: {} {} {} " + "assigned on-heap mem, off-heap mem, cpu: {} {} {}", td.getId(), requestedMemOnHeap, requestedMemOffHeap, requestedCpu, assignedMemOnHeap, assignedMemOffHeap, assignedCpu);
        //updating resources used for a topology
        this.schedulingState.cluster.setTopologyResources(td.getId(), resources);
        this.schedulingState.cluster.setWorkerResources(td.getId(), workerResources);
        return true;
    } else {
        LOG.warn("schedulerAssignmentMap for topo {} is null. This shouldn't happen!", td.getName());
        return false;
    }
}

Example 49 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class ResourceAwareScheduler method allocateResourceToSlot.

private WorkerSlot allocateResourceToSlot(TopologyDetails td, Collection<ExecutorDetails> executors, WorkerSlot slot) {
    double onHeapMem = 0.0;
    double offHeapMem = 0.0;
    double cpu = 0.0;
    for (ExecutorDetails exec : executors) {
        Double onHeapMemForExec = td.getOnHeapMemoryRequirement(exec);
        if (onHeapMemForExec != null) {
            onHeapMem += onHeapMemForExec;
        }
        Double offHeapMemForExec = td.getOffHeapMemoryRequirement(exec);
        if (offHeapMemForExec != null) {
            offHeapMem += offHeapMemForExec;
        }
        Double cpuForExec = td.getTotalCpuReqTask(exec);
        if (cpuForExec != null) {
            cpu += cpuForExec;
        }
    }
    return new WorkerSlot(slot.getNodeId(), slot.getPort(), onHeapMem, offHeapMem, cpu);
}

Example 50 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestDefaultResourceAwareStrategy method testDefaultResourceAwareStrategy.

/**
     * test if the scheduling logic for the DefaultResourceAwareStrategy is correct
     */
@Test
public void testDefaultResourceAwareStrategy() {
    int spoutParallelism = 1;
    int boltParallelism = 2;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new TestUtilsForResourceAwareScheduler.TestSpout(), spoutParallelism);
    builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(), boltParallelism).shuffleGrouping("spout");
    builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(), boltParallelism).shuffleGrouping("bolt-1");
    builder.setBolt("bolt-3", new TestUtilsForResourceAwareScheduler.TestBolt(), boltParallelism).shuffleGrouping("bolt-2");
    StormTopology stormToplogy = builder.createTopology();
    Config conf = new Config();
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 150.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1500.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
    conf.putAll(Utils.readDefaultConfig());
    conf.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    conf.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    conf.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    conf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 50.0);
    conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 250);
    conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 250);
    conf.put(Config.TOPOLOGY_PRIORITY, 0);
    conf.put(Config.TOPOLOGY_NAME, "testTopology");
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, TestUtilsForResourceAwareScheduler.genExecsAndComps(stormToplogy), this.currentTime);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo.getId(), topo);
    Topologies topologies = new Topologies(topoMap);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), conf);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(conf);
    rs.schedule(topologies, cluster);
    Map<String, List<String>> nodeToComps = new HashMap<String, List<String>>();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : cluster.getAssignments().get("testTopology-id").getExecutorToSlot().entrySet()) {
        WorkerSlot ws = entry.getValue();
        ExecutorDetails exec = entry.getKey();
        if (!nodeToComps.containsKey(ws.getNodeId())) {
            nodeToComps.put(ws.getNodeId(), new LinkedList<String>());
        }
        nodeToComps.get(ws.getNodeId()).add(topo.getExecutorToComponent().get(exec));
    }
    /**
         * check for correct scheduling
         * Since all the resource availabilites on nodes are the same in the beginining
         * DefaultResourceAwareStrategy can arbitrarily pick one thus we must find if a particular scheduling
         * exists on a node the the cluster.
         */
    //one node should have the below scheduling
    List<String> node1 = new LinkedList<>();
    node1.add("spout");
    node1.add("bolt-1");
    node1.add("bolt-2");
    Assert.assertTrue("Check DefaultResourceAwareStrategy scheduling", checkDefaultStrategyScheduling(nodeToComps, node1));
    //one node should have the below scheduling
    List<String> node2 = new LinkedList<>();
    node2.add("bolt-3");
    node2.add("bolt-1");
    node2.add("bolt-2");
    Assert.assertTrue("Check DefaultResourceAwareStrategy scheduling", checkDefaultStrategyScheduling(nodeToComps, node2));
    //one node should have the below scheduling
    List<String> node3 = new LinkedList<>();
    node3.add("bolt-3");
    Assert.assertTrue("Check DefaultResourceAwareStrategy scheduling", checkDefaultStrategyScheduling(nodeToComps, node3));
    //three used and one node should be empty
    Assert.assertEquals("only three nodes should be used", 3, nodeToComps.size());
}