Examples with ExecutorDetails org.apache.storm.scheduler.ExecutorDetails used on opensource projects

Example 56 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class NodeSorter method getScheduledExecCntByRackId.

private Map<String, AtomicInteger> getScheduledExecCntByRackId() {
    String topoId = topologyDetails.getId();
    SchedulerAssignment assignment = cluster.getAssignmentById(topoId);
    Map<String, AtomicInteger> scheduledCount = new HashMap<>();
    if (assignment != null) {
        for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> entry : assignment.getSlotToExecutors().entrySet()) {
            String superId = entry.getKey().getNodeId();
            String rackId = superIdToRack.get(superId);
            scheduledCount.computeIfAbsent(rackId, (rid) -> new AtomicInteger(0)).getAndAdd(entry.getValue().size());
        }
    }
    return scheduledCount;
}

Example 57 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class RasNode method freeSingleExecutor.

/**
 * frees a single executor.
 *
 * @param exec is the executor to free
 * @param topo the topology the executor is a part of
 */
public void freeSingleExecutor(ExecutorDetails exec, TopologyDetails topo) {
    Map<String, Collection<ExecutorDetails>> usedSlots = topIdToUsedSlots.get(topo.getId());
    if (usedSlots == null) {
        throw new IllegalArgumentException("Topology " + topo + " is not assigned");
    }
    WorkerSlot ws = null;
    Set<ExecutorDetails> updatedAssignment = new HashSet<>();
    for (Entry<String, Collection<ExecutorDetails>> entry : usedSlots.entrySet()) {
        if (entry.getValue().contains(exec)) {
            ws = slots.get(entry.getKey());
            updatedAssignment.addAll(entry.getValue());
            updatedAssignment.remove(exec);
            break;
        }
    }
    if (ws == null) {
        throw new IllegalArgumentException("Executor " + exec + " is not assinged on this node to " + topo);
    }
    free(ws);
    if (!updatedAssignment.isEmpty()) {
        assign(ws, topo, updatedAssignment);
    }
}

Example 58 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class RasNodes method getAllNodesFrom.

public static Map<String, RasNode> getAllNodesFrom(Cluster cluster) {
    // A map of node ids to node objects
    Map<String, RasNode> nodeIdToNode = new HashMap<>();
    // A map of assignments organized by node with the following format:
    // {nodeId -> {topologyId -> {workerId -> {execs}}}}
    Map<String, Map<String, Map<String, Collection<ExecutorDetails>>>> assignmentRelationshipMap = new HashMap<>();
    Map<String, Map<String, WorkerSlot>> workerIdToWorker = new HashMap<>();
    for (SchedulerAssignment assignment : cluster.getAssignments().values()) {
        String topId = assignment.getTopologyId();
        for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> entry : assignment.getSlotToExecutors().entrySet()) {
            WorkerSlot slot = entry.getKey();
            String nodeId = slot.getNodeId();
            if (!assignmentRelationshipMap.containsKey(nodeId)) {
                assignmentRelationshipMap.put(nodeId, new HashMap<>());
                workerIdToWorker.put(nodeId, new HashMap<>());
            }
            workerIdToWorker.get(nodeId).put(slot.getId(), slot);
            if (!assignmentRelationshipMap.get(nodeId).containsKey(topId)) {
                assignmentRelationshipMap.get(nodeId).put(topId, new HashMap<>());
            }
            if (!assignmentRelationshipMap.get(nodeId).get(topId).containsKey(slot.getId())) {
                assignmentRelationshipMap.get(nodeId).get(topId).put(slot.getId(), new LinkedList<>());
            }
            Collection<ExecutorDetails> execs = entry.getValue();
            assignmentRelationshipMap.get(nodeId).get(topId).get(slot.getId()).addAll(execs);
        }
    }
    for (SupervisorDetails sup : cluster.getSupervisors().values()) {
        // Initialize a worker slot for every port even if there is no assignment to it
        for (int port : sup.getAllPorts()) {
            WorkerSlot worker = new WorkerSlot(sup.getId(), port);
            if (!workerIdToWorker.containsKey(sup.getId())) {
                workerIdToWorker.put(sup.getId(), new HashMap<>());
            }
            if (!workerIdToWorker.get(sup.getId()).containsKey(worker.getId())) {
                workerIdToWorker.get(sup.getId()).put(worker.getId(), worker);
            }
        }
        nodeIdToNode.put(sup.getId(), new RasNode(sup.getId(), sup, cluster, workerIdToWorker.get(sup.getId()), assignmentRelationshipMap.get(sup.getId())));
    }
    // Add in supervisors that might have crashed but workers are still alive
    for (Map.Entry<String, Map<String, Map<String, Collection<ExecutorDetails>>>> entry : assignmentRelationshipMap.entrySet()) {
        String nodeId = entry.getKey();
        Map<String, Map<String, Collection<ExecutorDetails>>> assignments = entry.getValue();
        if (!nodeIdToNode.containsKey(nodeId)) {
            LOG.info("Found an assigned slot(s) on a dead supervisor {} with assignments {}", nodeId, assignments);
            nodeIdToNode.put(nodeId, new RasNode(nodeId, null, cluster, workerIdToWorker.get(nodeId), assignments));
        }
    }
    return nodeIdToNode;
}

Example 59 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class BaseResourceAwareStrategy method schedule.

/**
 * Note that this method is not thread-safe.
 * Several instance variables are generated from supplied
 * parameters. In addition, the following instance variables are set to complete scheduling:
 *  <li>{@link #searcherState}</li>
 *  <li>{@link #execSorter} to sort executors</li>
 *  <li>{@link #nodeSorter} to sort nodes</li>
 * <p>
 * Scheduling consists of three main steps:
 *  <li>{@link #prepareForScheduling(Cluster, TopologyDetails)}</li>
 *  <li>{@link #checkSchedulingFeasibility()}, and</li>
 *  <li>{@link #scheduleExecutorsOnNodes(List, Iterable)}</li>
 * </p><p>
 * The executors and nodes are sorted in the order most conducive to scheduling for the strategy.
 * Those interfaces may be overridden by subclasses using mutators:
 *  <li>{@link #setExecSorter(IExecSorter)} and</li>
 *  <li>{@link #setNodeSorter(INodeSorter)}</li>
 *</p>
 *
 * @param cluster on which executors will be scheduled.
 * @param td the topology to schedule for.
 * @return result of scheduling (success, failure, or null when interrupted).
 */
@Override
public SchedulingResult schedule(Cluster cluster, TopologyDetails td) {
    prepareForScheduling(cluster, td);
    // early detection of success or failure
    SchedulingResult earlyResult = checkSchedulingFeasibility();
    if (earlyResult != null) {
        return earlyResult;
    }
    LOG.debug("Topology {} {} Number of ExecutorsNeedScheduling: {}", topoName, topologyDetails.getId(), unassignedExecutors.size());
    // order executors to be scheduled
    List<ExecutorDetails> orderedExecutors = execSorter.sortExecutors(unassignedExecutors);
    Iterable<String> sortedNodes = null;
    if (!this.sortNodesForEachExecutor) {
        nodeSorter.prepare(null);
        sortedNodes = nodeSorter.sortAllNodes();
    }
    return scheduleExecutorsOnNodes(orderedExecutors, sortedNodes);
}

Example 60 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class BaseResourceAwareStrategy method createSearcherState.

/**
 * Create an instance of {@link SchedulingSearcherState}. This method is called by
 * {@link #prepareForScheduling(Cluster, TopologyDetails)} and depends on variables initialized therein prior.
 *
 * @return a new instance of {@link SchedulingSearcherState}.
 */
private SchedulingSearcherState createSearcherState() {
    Map<WorkerSlot, Map<String, Integer>> workerCompCnts = new HashMap<>();
    Map<RasNode, Map<String, Integer>> nodeCompCnts = new HashMap<>();
    // populate with existing assignments
    SchedulerAssignment existingAssignment = cluster.getAssignmentById(topologyDetails.getId());
    if (existingAssignment != null) {
        existingAssignment.getExecutorToSlot().forEach((exec, ws) -> {
            String compId = execToComp.get(exec);
            RasNode node = nodes.getNodeById(ws.getNodeId());
            Map<String, Integer> compCnts = nodeCompCnts.computeIfAbsent(node, (k) -> new HashMap<>());
            // increment
            compCnts.put(compId, compCnts.getOrDefault(compId, 0) + 1);
            // populate worker to comp assignments
            compCnts = workerCompCnts.computeIfAbsent(ws, (k) -> new HashMap<>());
            // increment
            compCnts.put(compId, compCnts.getOrDefault(compId, 0) + 1);
        });
    }
    LinkedList<ExecutorDetails> unassignedAckers = new LinkedList<>();
    if (compToExecs.containsKey(Acker.ACKER_COMPONENT_ID)) {
        for (ExecutorDetails acker : compToExecs.get(Acker.ACKER_COMPONENT_ID)) {
            if (unassignedExecutors.contains(acker)) {
                unassignedAckers.add(acker);
            }
        }
    }
    return new SchedulingSearcherState(workerCompCnts, nodeCompCnts, maxStateSearch, maxSchedulingTimeMs, new ArrayList<>(unassignedExecutors), unassignedAckers, topologyDetails, execToComp);
}