Examples with LogicalPlan edu.iu.dsc.tws.api.comms.LogicalPlan used on opensource projects

Example 6 with LogicalPlan

use of edu.iu.dsc.tws.api.comms.LogicalPlan in project twister2 by DSC-SPIDAL.

the class TaskPlanBuilder method build.

/**
 * Create a task plan based on the resource plan from resources and scheduled plan
 *
 * @param schedulePlan schedule plan
 * @param idGenerator global task id generator
 * @return the task plan
 */
public static LogicalPlan build(int workerID, List<JobMasterAPI.WorkerInfo> workerInfoList, TaskSchedulePlan schedulePlan, TaskIdGenerator idGenerator) {
    Set<WorkerSchedulePlan> cPlanList = schedulePlan.getContainers();
    Map<Integer, Set<Integer>> containersToTasks = new HashMap<>();
    Map<Integer, Set<Integer>> groupsToTasks = new HashMap<>();
    // we need to sort to keep the order
    workerInfoList.sort(Comparator.comparingInt(JobMasterAPI.WorkerInfo::getWorkerID));
    for (WorkerSchedulePlan c : cPlanList) {
        Set<TaskInstancePlan> tSet = c.getTaskInstances();
        Set<Integer> instances = new HashSet<>();
        for (TaskInstancePlan tPlan : tSet) {
            instances.add(idGenerator.generateGlobalTaskId(tPlan.getTaskId(), tPlan.getTaskIndex()));
        }
        containersToTasks.put(c.getContainerId(), instances);
    }
    Map<String, List<JobMasterAPI.WorkerInfo>> containersPerNode = new TreeMap<>();
    for (JobMasterAPI.WorkerInfo workerInfo : workerInfoList) {
        String name = Integer.toString(workerInfo.getWorkerID());
        List<JobMasterAPI.WorkerInfo> containerList;
        if (!containersPerNode.containsKey(name)) {
            containerList = new ArrayList<>();
            containersPerNode.put(name, containerList);
        } else {
            containerList = containersPerNode.get(name);
        }
        containerList.add(workerInfo);
    }
    Map<String, Set<Integer>> nodeToTasks = new HashMap<>();
    int i = 0;
    // we take each container as an executor
    for (Map.Entry<String, List<JobMasterAPI.WorkerInfo>> entry : containersPerNode.entrySet()) {
        Set<Integer> executorsOfGroup = new HashSet<>();
        for (JobMasterAPI.WorkerInfo workerInfo : entry.getValue()) {
            executorsOfGroup.add(workerInfo.getWorkerID());
            Set<Integer> tasksInNode = nodeToTasks.computeIfAbsent(workerInfo.getNodeInfo().getNodeIP(), k -> new HashSet<>());
            tasksInNode.addAll(containersToTasks.getOrDefault(workerInfo.getWorkerID(), Collections.emptySet()));
        }
        groupsToTasks.put(i, executorsOfGroup);
        i++;
    }
    return new LogicalPlan(containersToTasks, groupsToTasks, nodeToTasks, workerID);
}