Examples with ExecutorDetails org.apache.storm.scheduler.ExecutorDetails used on opensource projects

Example 16 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class NodeSorterHostProximity method getScheduledExecCntByRackId.

public Map<String, AtomicInteger> getScheduledExecCntByRackId() {
    SchedulerAssignment assignment = cluster.getAssignmentById(topologyDetails.getId());
    Map<String, AtomicInteger> scheduledCount = new HashMap<>();
    if (assignment != null) {
        for (Map.Entry<WorkerSlot, Collection<ExecutorDetails>> entry : assignment.getSlotToExecutors().entrySet()) {
            String superId = entry.getKey().getNodeId();
            String rackId = superIdToRack.get(superId);
            scheduledCount.computeIfAbsent(rackId, (rid) -> new AtomicInteger(0)).getAndAdd(entry.getValue().size());
        }
    }
    return scheduledCount;
}

Example 17 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class RasNode method wouldFit.

/**
 * Would scheduling exec in ws fit with the current resource constraints.
 *
 * @param ws   the slot to possibly put exec in
 * @param exec the executor to possibly place in ws
 * @param td   the topology exec is a part of
 * @return true if it would fit else false
 */
public boolean wouldFit(WorkerSlot ws, ExecutorDetails exec, TopologyDetails td) {
    assert nodeId.equals(ws.getNodeId()) : "Slot " + ws + " is not a part of this node " + nodeId;
    if (!isAlive || !cluster.wouldFit(ws, exec, td, getTotalAvailableResources(), td.getTopologyWorkerMaxHeapSize())) {
        return false;
    }
    boolean oneExecutorPerWorker = (Boolean) td.getConf().get(Config.TOPOLOGY_RAS_ONE_EXECUTOR_PER_WORKER);
    boolean oneComponentPerWorker = (Boolean) td.getConf().get(Config.TOPOLOGY_RAS_ONE_COMPONENT_PER_WORKER);
    if (oneExecutorPerWorker) {
        return !getUsedSlots(td.getId()).contains(ws);
    }
    if (oneComponentPerWorker) {
        Set<String> components = new HashSet<>();
        Map<String, Collection<ExecutorDetails>> topologyExecutors = topIdToUsedSlots.get(td.getId());
        if (topologyExecutors != null) {
            Collection<ExecutorDetails> slotExecs = topologyExecutors.get(ws.getId());
            if (slotExecs != null) {
                // components from WorkerSlot
                for (ExecutorDetails slotExec : slotExecs) {
                    components.add(td.getComponentFromExecutor(slotExec));
                }
                // component from exec
                components.add(td.getComponentFromExecutor(exec));
            }
        }
        return components.size() <= 1;
    }
    return true;
}

Example 18 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class SchedulingSearcherState method backtrack.

public void backtrack(Map<ExecutorDetails, String> execToComp, RasNode node, WorkerSlot workerSlot) {
    execIndex--;
    // when backtrack, we need to skip over the bound ackers
    while (execIndex >= 0 && boundAckers.contains(execs.get(execIndex))) {
        execIndex--;
    }
    if (execIndex < 0) {
        throw new IllegalStateException("Internal Error: Topology " + topoName + " exec index became negative");
    }
    numBacktrack++;
    ExecutorDetails exec = currentExec();
    String comp = execToComp.get(exec);
    LOG.trace("Topology {} Backtracking {} {} from {}", topoName, exec, comp, workerSlot);
    if (okToRemoveFromWorker[execIndex]) {
        Map<String, Integer> compToAssignmentCount = workerCompAssignmentCnts.get(workerSlot);
        // decrement worker assignment count
        compToAssignmentCount.put(comp, compToAssignmentCount.getOrDefault(comp, 0) - 1);
        if (compToAssignmentCount.get(comp) == 0) {
            compToAssignmentCount.remove(comp);
        }
        okToRemoveFromWorker[execIndex] = false;
    }
    if (okToRemoveFromNode[execIndex]) {
        Map<String, Integer> nodeToAssignmentCount = nodeCompAssignmentCnts.get(node);
        // decrement node assignment count
        nodeToAssignmentCount.put(comp, nodeToAssignmentCount.getOrDefault(comp, 0) - 1);
        if (nodeToAssignmentCount.get(comp) == 0) {
            nodeToAssignmentCount.remove(comp);
        }
        okToRemoveFromNode[execIndex] = false;
    }
    node.freeSingleExecutor(exec, td);
    // If this exec has bound ackers, we need to backtrack them as well
    if (execsWithBoundAckers.remove(exec)) {
        if (workerSlotToBoundAckers.containsKey(workerSlot)) {
            freeWorkerSlotWithBoundAckers(node, workerSlot);
        }
    }
}

Example 19 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class SchedulingSearcherState method logNodeCompAssignments.

/**
 * Use this method to log the current component assignments on the Node.
 * Useful for debugging and tests.
 */
public void logNodeCompAssignments() {
    if (nodeCompAssignmentCnts == null || nodeCompAssignmentCnts.isEmpty()) {
        LOG.info("Topology {} NodeCompAssignment is empty", topoName);
        return;
    }
    StringBuffer sb = new StringBuffer();
    int cntAllNodes = 0;
    int cntFilledNodes = 0;
    for (RasNode node : new TreeSet<>(nodeCompAssignmentCnts.keySet())) {
        cntAllNodes++;
        Map<String, Integer> oneMap = nodeCompAssignmentCnts.get(node);
        if (oneMap.isEmpty()) {
            continue;
        }
        cntFilledNodes++;
        String oneMapJoined = oneMap.entrySet().stream().map(e -> String.format("%s: %s", e.getKey(), e.getValue())).collect(Collectors.joining(","));
        sb.append(String.format("\n\t(%d) Node %s: %s", cntFilledNodes, node.getId(), oneMapJoined));
    }
    LOG.info("Topology {} NodeCompAssignments available for {} of {} nodes {}", topoName, cntFilledNodes, cntAllNodes, sb);
    LOG.info("Topology {} Executors assignments attempted (cnt={}) are: \n\t{}", topoName, execs.size(), execs.stream().map(ExecutorDetails::toString).collect(Collectors.joining(",")));
}

Example 20 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class SchedulingSearcherState method assignSingleBoundAcker.

/**
 * <p>
 * Remove the head of unassigned ackers and attempt to assign it to a workerSlot as a bound acker.
 * </p>
 *
 * @param node          RasNode on which to schedule.
 * @param workerSlot    WorkerSlot on which to schedule.
 */
public void assignSingleBoundAcker(RasNode node, WorkerSlot workerSlot) {
    if (unassignedAckers.isEmpty()) {
        String msg = String.format("No more available ackers to assign for the new worker: %s of topology: %s", workerSlot, topoName);
        throw new IllegalStateException(msg);
    }
    ExecutorDetails acker = unassignedAckers.removeFirst();
    node.assignSingleExecutor(workerSlot, acker, td);
    if (!workerSlotToBoundAckers.containsKey(workerSlot)) {
        workerSlotToBoundAckers.put(workerSlot, new ArrayList<>());
    }
    workerSlotToBoundAckers.get(workerSlot).add(acker);
    boundAckers.add(acker);
    // bound ackers should not violate constraint solver
    String ackerCompId = Acker.ACKER_COMPONENT_ID;
    Map<String, Integer> compToAssignmentCount = workerCompAssignmentCnts.computeIfAbsent(workerSlot, (k) -> new HashMap<>());
    // increment worker assignment count
    compToAssignmentCount.put(ackerCompId, compToAssignmentCount.getOrDefault(ackerCompId, 0) + 1);
    Map<String, Integer> nodeToAssignmentCount = nodeCompAssignmentCnts.computeIfAbsent(node, (k) -> new HashMap<>());
    // increment node assignment count
    nodeToAssignmentCount.put(ackerCompId, nodeToAssignmentCount.getOrDefault(ackerCompId, 0) + 1);
}