Examples with ExecutorDetails org.apache.storm.scheduler.ExecutorDetails used on opensource projects

Example 11 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class Nimbus method computeTopoToExecToNodePort.

/**
     * convert {topology-id -> SchedulerAssignment} to
     *         {topology-id -> {executor [node port]}}
     * @return
     */
private static Map<String, Map<List<Long>, List<Object>>> computeTopoToExecToNodePort(Map<String, SchedulerAssignment> schedAssignments) {
    Map<String, Map<List<Long>, List<Object>>> ret = new HashMap<>();
    for (Entry<String, SchedulerAssignment> schedEntry : schedAssignments.entrySet()) {
        Map<List<Long>, List<Object>> execToNodePort = new HashMap<>();
        for (Entry<ExecutorDetails, WorkerSlot> execAndNodePort : schedEntry.getValue().getExecutorToSlot().entrySet()) {
            ExecutorDetails exec = execAndNodePort.getKey();
            WorkerSlot slot = execAndNodePort.getValue();
            List<Long> listExec = new ArrayList<>(2);
            listExec.add((long) exec.getStartTask());
            listExec.add((long) exec.getEndTask());
            List<Object> nodePort = new ArrayList<>(2);
            nodePort.add(slot.getNodeId());
            nodePort.add((long) slot.getPort());
            execToNodePort.put(listExec, nodePort);
        }
        ret.put(schedEntry.getKey(), execToNodePort);
    }
    return ret;
}

Example 12 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class ExecSorterByConnectionCount method sortExecutors.

/**
 * Order executors based on how many in and out connections it will potentially need to make, in descending order. First order
 * components by the number of in and out connections it will have.  Then iterate through the sorted list of components. For each
 * component sort the neighbors of that component by how many connections it will have to make with that component.
 * Add an executor from this component and then from each neighboring component in sorted order. Do this until there is
 * nothing left to schedule. Then add back executors not accounted for - which are system executors.
 *
 * @param unassignedExecutors an unmodifiable set of executors that need to be scheduled.
 * @return a list of executors in sorted order for scheduling.
 */
public List<ExecutorDetails> sortExecutors(Set<ExecutorDetails> unassignedExecutors) {
    // excludes system components
    Map<String, Component> componentMap = topologyDetails.getUserTopolgyComponents();
    // in insert order
    LinkedHashSet<ExecutorDetails> orderedExecutorSet = new LinkedHashSet<>();
    Map<String, Queue<ExecutorDetails>> compToExecsToSchedule = new HashMap<>();
    for (Component component : componentMap.values()) {
        compToExecsToSchedule.put(component.getId(), new LinkedList<>());
        for (ExecutorDetails exec : component.getExecs()) {
            if (unassignedExecutors.contains(exec)) {
                compToExecsToSchedule.get(component.getId()).add(exec);
            }
        }
    }
    Set<Component> sortedComponents = sortComponents(componentMap);
    sortedComponents.addAll(componentMap.values());
    for (Component currComp : sortedComponents) {
        Map<String, Component> neighbors = new HashMap<>();
        for (String compId : Sets.union(currComp.getChildren(), currComp.getParents())) {
            neighbors.put(compId, componentMap.get(compId));
        }
        Set<Component> sortedNeighbors = sortNeighbors(currComp, neighbors);
        Queue<ExecutorDetails> currCompExecsToSched = compToExecsToSchedule.get(currComp.getId());
        boolean flag;
        do {
            flag = false;
            if (!currCompExecsToSched.isEmpty()) {
                orderedExecutorSet.add(currCompExecsToSched.poll());
                flag = true;
            }
            for (Component neighborComp : sortedNeighbors) {
                Queue<ExecutorDetails> neighborCompExesToSched = compToExecsToSchedule.get(neighborComp.getId());
                if (!neighborCompExesToSched.isEmpty()) {
                    orderedExecutorSet.add(neighborCompExesToSched.poll());
                    flag = true;
                }
            }
        } while (flag);
    }
    // add executors not in sorted list - which may be system executors
    orderedExecutorSet.addAll(unassignedExecutors);
    return new LinkedList<>(orderedExecutorSet);
}

Example 13 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class NodeSorter method sortObjectResourcesCommon.

/**
 * Sort objects by the following three criteria.
 *
 * <li>
 *     The number executors of the topology that needs to be scheduled is already on the object (node or rack)
 *     in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on
 *     the same object (node or rack) as the existing executors of the topology.
 * </li>
 *
 * <li>
 *     The subordinate/subservient resource availability percentage of a rack in descending order We calculate the
 *     resource availability percentage by dividing the resource availability of the object (node or rack) by the
 *     resource availability of the entire rack or cluster depending on if object references a node or a rack.
 *     How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is
 *     requested by the executor that the sorting is being done for and pulls it down if it is not.
 *     By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned
 *     above will be ranked after racks that have more balanced resource availability and nodes or racks that have
 *     resources that are not requested will be ranked below . So we will be less likely to pick a rack that
 *     have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor.
 *     This is similar to logic used {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 * </li>
 *
 * <li>
 *     The tie between two nodes with same resource availability is broken by using the node with lower minimum
 *     percentage used. This comparison was used in {@link #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)}
 *     but here it is made subservient to modified resource availbility used in
 *     {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 *
 * </li>
 *
 * @param allResources         contains all individual ObjectResources as well as cumulative stats
 * @param exec                 executor for which the sorting is done
 * @param existingScheduleFunc a function to get existing executors already scheduled on this object
 * @return a sorted list of ObjectResources
 */
private List<ObjectResourcesItem> sortObjectResourcesCommon(final ObjectResourcesSummary allResources, final ExecutorDetails exec, final ExistingScheduleFunc existingScheduleFunc) {
    // Copy and modify allResources
    ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources);
    final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall();
    final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails.getTotalResources(exec) : null;
    affinityBasedAllResources.getObjectResources().forEach(x -> {
        x.minResourcePercent = availableResourcesOverall.calculateMinPercentageUsedBy(x.availableResources);
        if (requestedResources != null) {
            // negate unrequested resources
            x.availableResources.updateForRareResourceAffinity(requestedResources);
        }
        x.avgResourcePercent = availableResourcesOverall.calculateAveragePercentageUsedBy(x.availableResources);
        LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", x.id, x.minResourcePercent, x.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(x.id));
    });
    // Use the following comparator to return a sorted set
    List<ObjectResourcesItem> sortedObjectResources = new ArrayList();
    Comparator<ObjectResourcesItem> comparator = (o1, o2) -> {
        int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
        int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
        if (execsScheduled1 > execsScheduled2) {
            return -1;
        } else if (execsScheduled1 < execsScheduled2) {
            return 1;
        }
        double o1Avg = o1.avgResourcePercent;
        double o2Avg = o2.avgResourcePercent;
        if (o1Avg > o2Avg) {
            return -1;
        } else if (o1Avg < o2Avg) {
            return 1;
        }
        if (o1.minResourcePercent > o2.minResourcePercent) {
            return -1;
        } else if (o1.minResourcePercent < o2.minResourcePercent) {
            return 1;
        }
        return o1.id.compareTo(o2.id);
    };
    sortedObjectResources.addAll(affinityBasedAllResources.getObjectResources());
    sortedObjectResources.sort(comparator);
    LOG.debug("Sorted Object Resources: {}", sortedObjectResources);
    return sortedObjectResources;
}

Example 14 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class ConstraintSolverStrategy method validateSolution.

/**
 * Determines if a scheduling is valid and all constraints are satisfied (for use in testing).
 * This is done in three steps.
 *
 * <li>Check if nodeCoLocationCnt-constraints are satisfied. Some components may allow only a certain number of
 * executors to exist on the same node {@link ConstraintSolverConfig#getMaxNodeCoLocationCnts()}.
 * </li>
 *
 * <li>
 * Check if incompatibility-constraints are satisfied. Incompatible components
 * {@link ConstraintSolverConfig#getIncompatibleComponentSets()} should not be put on the same worker.
 * </li>
 *
 * <li>
 * Check if CPU and Memory resources do not exceed availability on the node and total matches what is expected
 * when fully scheduled.
 * </li>
 *
 * @param cluster on which scheduling was done.
 * @param topo TopologyDetails being scheduled.
 * @return true if solution is valid, false otherwise.
 */
@VisibleForTesting
public static boolean validateSolution(Cluster cluster, TopologyDetails topo) {
    assert (cluster.getAssignmentById(topo.getId()) != null);
    LOG.debug("Checking for a valid scheduling for topology {}...", topo.getName());
    ConstraintSolverConfig constraintSolverConfig = new ConstraintSolverConfig(topo);
    // First check NodeCoLocationCnt constraints
    Map<ExecutorDetails, String> execToComp = topo.getExecutorToComponent();
    // this is the critical count
    Map<String, Map<String, Integer>> nodeCompMap = new HashMap<>();
    Map<WorkerSlot, RasNode> workerToNodes = new HashMap<>();
    RasNodes.getAllNodesFrom(cluster).values().forEach(node -> node.getUsedSlots().forEach(workerSlot -> workerToNodes.put(workerSlot, node)));
    List<String> errors = new ArrayList<>();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : cluster.getAssignmentById(topo.getId()).getExecutorToSlot().entrySet()) {
        ExecutorDetails exec = entry.getKey();
        String comp = execToComp.get(exec);
        WorkerSlot worker = entry.getValue();
        RasNode node = workerToNodes.get(worker);
        String nodeId = node.getId();
        if (!constraintSolverConfig.getMaxNodeCoLocationCnts().containsKey(comp)) {
            continue;
        }
        int allowedColocationMaxCnt = constraintSolverConfig.getMaxNodeCoLocationCnts().get(comp);
        Map<String, Integer> oneNodeCompMap = nodeCompMap.computeIfAbsent(nodeId, (k) -> new HashMap<>());
        oneNodeCompMap.put(comp, oneNodeCompMap.getOrDefault(comp, 0) + 1);
        if (allowedColocationMaxCnt < oneNodeCompMap.get(comp)) {
            String err = String.format("MaxNodeCoLocation: Component %s (exec=%s) on node %s, cnt %d > allowed %d", comp, exec, nodeId, oneNodeCompMap.get(comp), allowedColocationMaxCnt);
            errors.add(err);
        }
    }
    // Second check IncompatibileComponent Constraints
    Map<WorkerSlot, Set<String>> workerCompMap = new HashMap<>();
    cluster.getAssignmentById(topo.getId()).getExecutorToSlot().forEach((exec, worker) -> {
        String comp = execToComp.get(exec);
        workerCompMap.computeIfAbsent(worker, (k) -> new HashSet<>()).add(comp);
    });
    for (Map.Entry<WorkerSlot, Set<String>> entry : workerCompMap.entrySet()) {
        Set<String> comps = entry.getValue();
        for (String comp1 : comps) {
            for (String comp2 : comps) {
                if (!comp1.equals(comp2) && constraintSolverConfig.getIncompatibleComponentSets().containsKey(comp1) && constraintSolverConfig.getIncompatibleComponentSets().get(comp1).contains(comp2)) {
                    String err = String.format("IncompatibleComponents: %s and %s on WorkerSlot: %s", comp1, comp2, entry.getKey());
                    errors.add(err);
                }
            }
        }
    }
    // Third check resources
    SchedulerAssignment schedulerAssignment = cluster.getAssignmentById(topo.getId());
    Map<ExecutorDetails, WorkerSlot> execToWorker = new HashMap<>();
    if (schedulerAssignment.getExecutorToSlot() != null) {
        execToWorker.putAll(schedulerAssignment.getExecutorToSlot());
    }
    Map<String, RasNode> nodes = RasNodes.getAllNodesFrom(cluster);
    Map<RasNode, Collection<ExecutorDetails>> nodeToExecs = new HashMap<>();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : execToWorker.entrySet()) {
        ExecutorDetails exec = entry.getKey();
        WorkerSlot worker = entry.getValue();
        RasNode node = nodes.get(worker.getNodeId());
        if (node.getAvailableMemoryResources() < 0.0) {
            String err = String.format("Resource Exhausted: Found node %s with negative available memory %,.2f", node.getId(), node.getAvailableMemoryResources());
            errors.add(err);
            continue;
        }
        if (node.getAvailableCpuResources() < 0.0) {
            String err = String.format("Resource Exhausted: Found node %s with negative available CPU %,.2f", node.getId(), node.getAvailableCpuResources());
            errors.add(err);
            continue;
        }
        nodeToExecs.computeIfAbsent(node, (k) -> new HashSet<>()).add(exec);
    }
    for (Map.Entry<RasNode, Collection<ExecutorDetails>> entry : nodeToExecs.entrySet()) {
        RasNode node = entry.getKey();
        Collection<ExecutorDetails> execs = entry.getValue();
        double cpuUsed = 0.0;
        double memoryUsed = 0.0;
        for (ExecutorDetails exec : execs) {
            cpuUsed += topo.getTotalCpuReqTask(exec);
            memoryUsed += topo.getTotalMemReqTask(exec);
        }
        if (node.getAvailableCpuResources() != (node.getTotalCpuResources() - cpuUsed)) {
            String err = String.format("Incorrect CPU Resources: Node %s CPU available is %,.2f, expected %,.2f, " + "Executors scheduled on node: %s", node.getId(), node.getAvailableCpuResources(), (node.getTotalCpuResources() - cpuUsed), execs);
            errors.add(err);
        }
        if (node.getAvailableMemoryResources() != (node.getTotalMemoryResources() - memoryUsed)) {
            String err = String.format("Incorrect Memory Resources: Node %s Memory available is %,.2f, expected %,.2f, " + "Executors scheduled on node: %s", node.getId(), node.getAvailableMemoryResources(), (node.getTotalMemoryResources() - memoryUsed), execs);
            errors.add(err);
        }
    }
    if (!errors.isEmpty()) {
        LOG.error("Topology {} solution is invalid\n\t{}", topo.getName(), String.join("\n\t", errors));
    }
    return errors.isEmpty();
}

Example 15 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class NodeSorterHostProximity method sortObjectResourcesCommon.

/**
 * Sort objects by the following three criteria.
 *
 * <li>
 *     The number executors of the topology that needs to be scheduled is already on the object (node or rack)
 *     in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on
 *     the same object (node or rack) as the existing executors of the topology.
 * </li>
 *
 * <li>
 *     The subordinate/subservient resource availability percentage of a rack in descending order We calculate the
 *     resource availability percentage by dividing the resource availability of the object (node or rack) by the
 *     resource availability of the entire rack or cluster depending on if object references a node or a rack.
 *     How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is
 *     requested by the executor that the sorting is being done for and pulls it down if it is not.
 *     By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned
 *     above will be ranked after racks that have more balanced resource availability and nodes or racks that have
 *     resources that are not requested will be ranked below . So we will be less likely to pick a rack that
 *     have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor.
 *     This is similar to logic used {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 * </li>
 *
 * <li>
 *     The tie between two nodes with same resource availability is broken by using the node with lower minimum
 *     percentage used. This comparison was used in {@link #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)}
 *     but here it is made subservient to modified resource availbility used in
 *     {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 *
 * </li>
 *
 * @param allResources         contains all individual ObjectResources as well as cumulative stats
 * @param exec                 executor for which the sorting is done
 * @param existingScheduleFunc a function to get existing executors already scheduled on this object
 * @return an {@link Iterable} of sorted {@link ObjectResourcesItem}
 */
private Iterable<ObjectResourcesItem> sortObjectResourcesCommon(final ObjectResourcesSummary allResources, final ExecutorDetails exec, final ExistingScheduleFunc existingScheduleFunc) {
    // Copy and modify allResources
    ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources);
    final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall();
    final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails.getTotalResources(exec) : null;
    affinityBasedAllResources.getObjectResources().forEach(x -> {
        if (requestedResources != null) {
            // negate unrequested resources
            x.availableResources.updateForRareResourceAffinity(requestedResources);
        }
        x.minResourcePercent = availableResourcesOverall.calculateMinPercentageUsedBy(x.availableResources);
        x.avgResourcePercent = availableResourcesOverall.calculateAveragePercentageUsedBy(x.availableResources);
        LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", x.id, x.minResourcePercent, x.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(x.id));
    });
    // Use the following comparator to sort
    Comparator<ObjectResourcesItem> comparator = (o1, o2) -> {
        int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
        int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
        if (execsScheduled1 > execsScheduled2) {
            return -1;
        } else if (execsScheduled1 < execsScheduled2) {
            return 1;
        }
        double o1Avg = o1.avgResourcePercent;
        double o2Avg = o2.avgResourcePercent;
        if (o1Avg > o2Avg) {
            return -1;
        } else if (o1Avg < o2Avg) {
            return 1;
        }
        if (o1.minResourcePercent > o2.minResourcePercent) {
            return -1;
        } else if (o1.minResourcePercent < o2.minResourcePercent) {
            return 1;
        }
        return o1.id.compareTo(o2.id);
    };
    TreeSet<ObjectResourcesItem> sortedObjectResources = new TreeSet(comparator);
    sortedObjectResources.addAll(affinityBasedAllResources.getObjectResources());
    LOG.debug("Sorted Object Resources: {}", sortedObjectResources);
    return sortedObjectResources;
}