Examples with Component org.apache.storm.scheduler.resource.Component used on opensource projects

Example 1 with Component

use of org.apache.storm.scheduler.resource.Component in project storm by apache.

the class DefaultResourceAwareStrategy method orderExecutors.

/**
     * Order executors based on how many in and out connections it will potentially need to make.
     * First order components by the number of in and out connections it will have.  Then iterate through the sorted list of components.
     * For each component sort the neighbors of that component by how many connections it will have to make with that component.
     * Add an executor from this component and then from each neighboring component in sorted order.  Do this until there is nothing left to schedule
     *
     * @param td                  The topology the executors belong to
     * @param unassignedExecutors a collection of unassigned executors that need to be unassigned. Should only try to assign executors from this list
     * @return a list of executors in sorted order
     */
private List<ExecutorDetails> orderExecutors(TopologyDetails td, Collection<ExecutorDetails> unassignedExecutors) {
    Map<String, Component> componentMap = td.getComponents();
    List<ExecutorDetails> execsScheduled = new LinkedList<>();
    Map<String, Queue<ExecutorDetails>> compToExecsToSchedule = new HashMap<>();
    for (Component component : componentMap.values()) {
        compToExecsToSchedule.put(component.id, new LinkedList<ExecutorDetails>());
        for (ExecutorDetails exec : component.execs) {
            if (unassignedExecutors.contains(exec)) {
                compToExecsToSchedule.get(component.id).add(exec);
            }
        }
    }
    Set<Component> sortedComponents = sortComponents(componentMap);
    sortedComponents.addAll(componentMap.values());
    for (Component currComp : sortedComponents) {
        Map<String, Component> neighbors = new HashMap<String, Component>();
        for (String compId : (List<String>) ListUtils.union(currComp.children, currComp.parents)) {
            neighbors.put(compId, componentMap.get(compId));
        }
        Set<Component> sortedNeighbors = sortNeighbors(currComp, neighbors);
        Queue<ExecutorDetails> currCompExesToSched = compToExecsToSchedule.get(currComp.id);
        boolean flag = false;
        do {
            flag = false;
            if (!currCompExesToSched.isEmpty()) {
                execsScheduled.add(currCompExesToSched.poll());
                flag = true;
            }
            for (Component neighborComp : sortedNeighbors) {
                Queue<ExecutorDetails> neighborCompExesToSched = compToExecsToSchedule.get(neighborComp.id);
                if (!neighborCompExesToSched.isEmpty()) {
                    execsScheduled.add(neighborCompExesToSched.poll());
                    flag = true;
                }
            }
        } while (flag);
    }
    return execsScheduled;
}

Example 2 with Component

use of org.apache.storm.scheduler.resource.Component in project storm by apache.

the class DefaultResourceAwareStrategy method schedule.

public SchedulingResult schedule(TopologyDetails td) {
    if (_nodes.getNodes().size() <= 0) {
        LOG.warn("No available nodes to schedule tasks on!");
        return SchedulingResult.failure(SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, "No available nodes to schedule tasks on!");
    }
    Collection<ExecutorDetails> unassignedExecutors = new HashSet<ExecutorDetails>(_cluster.getUnassignedExecutors(td));
    Map<WorkerSlot, Collection<ExecutorDetails>> schedulerAssignmentMap = new HashMap<>();
    LOG.debug("ExecutorsNeedScheduling: {}", unassignedExecutors);
    Collection<ExecutorDetails> scheduledTasks = new ArrayList<>();
    List<Component> spouts = this.getSpouts(td);
    if (spouts.size() == 0) {
        LOG.error("Cannot find a Spout!");
        return SchedulingResult.failure(SchedulingStatus.FAIL_INVALID_TOPOLOGY, "Cannot find a Spout!");
    }
    //order executors to be scheduled
    List<ExecutorDetails> orderedExecutors = orderExecutors(td, unassignedExecutors);
    Collection<ExecutorDetails> executorsNotScheduled = new HashSet<>(unassignedExecutors);
    for (ExecutorDetails exec : orderedExecutors) {
        LOG.debug("\n\nAttempting to schedule: {} of component {}[ REQ {} ]", exec, td.getExecutorToComponent().get(exec), td.getTaskResourceReqList(exec));
        scheduleExecutor(exec, td, schedulerAssignmentMap, scheduledTasks);
    }
    executorsNotScheduled.removeAll(scheduledTasks);
    LOG.debug("/* Scheduling left over task (most likely sys tasks) */");
    // schedule left over system tasks
    for (ExecutorDetails exec : executorsNotScheduled) {
        scheduleExecutor(exec, td, schedulerAssignmentMap, scheduledTasks);
    }
    SchedulingResult result;
    executorsNotScheduled.removeAll(scheduledTasks);
    if (executorsNotScheduled.size() > 0) {
        LOG.error("Not all executors successfully scheduled: {}", executorsNotScheduled);
        schedulerAssignmentMap = null;
        result = SchedulingResult.failure(SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES, (td.getExecutors().size() - unassignedExecutors.size()) + "/" + td.getExecutors().size() + " executors scheduled");
    } else {
        LOG.debug("All resources successfully scheduled!");
        result = SchedulingResult.successWithMsg(schedulerAssignmentMap, "Fully Scheduled by DefaultResourceAwareStrategy");
    }
    if (schedulerAssignmentMap == null) {
        LOG.error("Topology {} not successfully scheduled!", td.getId());
    }
    return result;
}

Example 3 with Component

use of org.apache.storm.scheduler.resource.Component in project storm by apache.

the class TopologyDetails method getComponents.

/**
     * Returns a representation of the non-system components of the topology graph
     * Each Component object in the returning map is populated with the list of its
     * parents, children and execs assigned to that component.
     * @return a map of components
     */
public Map<String, Component> getComponents() {
    Map<String, Component> all_comp = new HashMap<>();
    StormTopology storm_topo = this.topology;
    // spouts
    if (storm_topo.get_spouts() != null) {
        for (Map.Entry<String, SpoutSpec> spoutEntry : storm_topo.get_spouts().entrySet()) {
            if (!Utils.isSystemId(spoutEntry.getKey())) {
                Component newComp;
                if (all_comp.containsKey(spoutEntry.getKey())) {
                    newComp = all_comp.get(spoutEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                } else {
                    newComp = new Component(spoutEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                    all_comp.put(spoutEntry.getKey(), newComp);
                }
                newComp.type = Component.ComponentType.SPOUT;
                for (Map.Entry<GlobalStreamId, Grouping> spoutInput : spoutEntry.getValue().get_common().get_inputs().entrySet()) {
                    newComp.parents.add(spoutInput.getKey().get_componentId());
                    if (!all_comp.containsKey(spoutInput.getKey().get_componentId())) {
                        all_comp.put(spoutInput.getKey().get_componentId(), new Component(spoutInput.getKey().get_componentId()));
                    }
                    all_comp.get(spoutInput.getKey().get_componentId()).children.add(spoutEntry.getKey());
                }
            }
        }
    }
    // bolts
    if (storm_topo.get_bolts() != null) {
        for (Map.Entry<String, Bolt> boltEntry : storm_topo.get_bolts().entrySet()) {
            if (!Utils.isSystemId(boltEntry.getKey())) {
                Component newComp;
                if (all_comp.containsKey(boltEntry.getKey())) {
                    newComp = all_comp.get(boltEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                } else {
                    newComp = new Component(boltEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                    all_comp.put(boltEntry.getKey(), newComp);
                }
                newComp.type = Component.ComponentType.BOLT;
                for (Map.Entry<GlobalStreamId, Grouping> boltInput : boltEntry.getValue().get_common().get_inputs().entrySet()) {
                    newComp.parents.add(boltInput.getKey().get_componentId());
                    if (!all_comp.containsKey(boltInput.getKey().get_componentId())) {
                        all_comp.put(boltInput.getKey().get_componentId(), new Component(boltInput.getKey().get_componentId()));
                    }
                    all_comp.get(boltInput.getKey().get_componentId()).children.add(boltEntry.getKey());
                }
            }
        }
    }
    return all_comp;
}