Search in sources :

Example 1 with Acker

use of org.apache.storm.daemon.Acker in project storm by apache.

the class BaseResourceAwareStrategy method createSearcherState.

/**
 * Create an instance of {@link SchedulingSearcherState}. This method is called by
 * {@link #prepareForScheduling(Cluster, TopologyDetails)} and depends on variables initialized therein prior.
 *
 * @return a new instance of {@link SchedulingSearcherState}.
 */
private SchedulingSearcherState createSearcherState() {
    Map<WorkerSlot, Map<String, Integer>> workerCompCnts = new HashMap<>();
    Map<RasNode, Map<String, Integer>> nodeCompCnts = new HashMap<>();
    // populate with existing assignments
    SchedulerAssignment existingAssignment = cluster.getAssignmentById(topologyDetails.getId());
    if (existingAssignment != null) {
        existingAssignment.getExecutorToSlot().forEach((exec, ws) -> {
            String compId = execToComp.get(exec);
            RasNode node = nodes.getNodeById(ws.getNodeId());
            Map<String, Integer> compCnts = nodeCompCnts.computeIfAbsent(node, (k) -> new HashMap<>());
            // increment
            compCnts.put(compId, compCnts.getOrDefault(compId, 0) + 1);
            // populate worker to comp assignments
            compCnts = workerCompCnts.computeIfAbsent(ws, (k) -> new HashMap<>());
            // increment
            compCnts.put(compId, compCnts.getOrDefault(compId, 0) + 1);
        });
    }
    LinkedList<ExecutorDetails> unassignedAckers = new LinkedList<>();
    if (compToExecs.containsKey(Acker.ACKER_COMPONENT_ID)) {
        for (ExecutorDetails acker : compToExecs.get(Acker.ACKER_COMPONENT_ID)) {
            if (unassignedExecutors.contains(acker)) {
                unassignedAckers.add(acker);
            }
        }
    }
    return new SchedulingSearcherState(workerCompCnts, nodeCompCnts, maxStateSearch, maxSchedulingTimeMs, new ArrayList<>(unassignedExecutors), unassignedAckers, topologyDetails, execToComp);
}
Also used : Acker(org.apache.storm.daemon.Acker) IExecSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.IExecSorter) ExecSorterByConnectionCount(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.ExecSorterByConnectionCount) RasNode(org.apache.storm.scheduler.resource.RasNode) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) NodeSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorter) RasNodes(org.apache.storm.scheduler.resource.RasNodes) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ExecSorterByProximity(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.ExecSorterByProximity) DaemonConfig(org.apache.storm.DaemonConfig) Map(java.util.Map) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) LinkedList(java.util.LinkedList) NodeSorterHostProximity(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorterHostProximity) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) Logger(org.slf4j.Logger) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) Set(java.util.Set) INodeSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.INodeSorter) SchedulingStatus(org.apache.storm.scheduler.resource.SchedulingStatus) Cluster(org.apache.storm.scheduler.Cluster) Time(org.apache.storm.utils.Time) SchedulingResult(org.apache.storm.scheduler.resource.SchedulingResult) List(java.util.List) ObjectReader(org.apache.storm.utils.ObjectReader) Config(org.apache.storm.Config) Collections(java.util.Collections) ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) LinkedList(java.util.LinkedList) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) RasNode(org.apache.storm.scheduler.resource.RasNode) HashMap(java.util.HashMap) Map(java.util.Map)

Aggregations

ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 LinkedList (java.util.LinkedList)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 Config (org.apache.storm.Config)1 DaemonConfig (org.apache.storm.DaemonConfig)1 Acker (org.apache.storm.daemon.Acker)1 Cluster (org.apache.storm.scheduler.Cluster)1 ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)1 SchedulerAssignment (org.apache.storm.scheduler.SchedulerAssignment)1 TopologyDetails (org.apache.storm.scheduler.TopologyDetails)1 WorkerSlot (org.apache.storm.scheduler.WorkerSlot)1 RasNode (org.apache.storm.scheduler.resource.RasNode)1 RasNodes (org.apache.storm.scheduler.resource.RasNodes)1 SchedulingResult (org.apache.storm.scheduler.resource.SchedulingResult)1 SchedulingStatus (org.apache.storm.scheduler.resource.SchedulingStatus)1