Examples with WorkerSchedulePlan edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan used on opensource projects

Example 16 with WorkerSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 *
 * @param computeGraph
 * @param workerPlan   worker plan
 * @return
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph computeGraph, WorkerPlan workerPlan) {
    // Allocate the task instances into the containers/workers
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    // To get the vertex set from the Collectible Name Settaskgraph
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(computeGraph.getTaskVertexSet());
    // Allocate the task instances into the logical containers.
    Map<Integer, List<TaskInstanceId>> batchContainerInstanceMap = batchSchedulingAlgorithm(computeGraph, workerPlan.getNumberOfWorkers());
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(batchContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(batchContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(batchContainerInstanceMap, taskVertexSet);
    for (int containerId : batchContainerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = batchContainerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        // Create the container resource value based on the worker plan
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
        }
        // Schedule the task instance plan into the task container plan.
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new LinkedHashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
        if (dependentGraphs && index == 0) {
            workerIdList.add(containerId);
        }
    }
    index++;
    TaskSchedulePlan taskSchedulePlan = new TaskSchedulePlan(0, workerSchedulePlans);
    if (workerId == 0) {
        Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            Integer integer = entry.getKey();
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            LOG.fine("Graph Name:" + computeGraph.getGraphName() + "\tcontainer id:" + integer);
            for (TaskInstancePlan ip : containerPlanTaskInstances) {
                LOG.fine("Task Id:" + ip.getTaskId() + "\tIndex" + ip.getTaskIndex() + "\tName:" + ip.getTaskName());
            }
        }
    }
    return taskSchedulePlan;
}

Example 17 with WorkerSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan in project twister2 by DSC-SPIDAL.

the class TaskPlanBuilder method build.

/**
 * Create a task plan based on the resource plan from resources and scheduled plan
 *
 * @param schedulePlan schedule plan
 * @param idGenerator global task id generator
 * @return the task plan
 */
public static LogicalPlan build(int workerID, List<JobMasterAPI.WorkerInfo> workerInfoList, TaskSchedulePlan schedulePlan, TaskIdGenerator idGenerator) {
    Set<WorkerSchedulePlan> cPlanList = schedulePlan.getContainers();
    Map<Integer, Set<Integer>> containersToTasks = new HashMap<>();
    Map<Integer, Set<Integer>> groupsToTasks = new HashMap<>();
    // we need to sort to keep the order
    workerInfoList.sort(Comparator.comparingInt(JobMasterAPI.WorkerInfo::getWorkerID));
    for (WorkerSchedulePlan c : cPlanList) {
        Set<TaskInstancePlan> tSet = c.getTaskInstances();
        Set<Integer> instances = new HashSet<>();
        for (TaskInstancePlan tPlan : tSet) {
            instances.add(idGenerator.generateGlobalTaskId(tPlan.getTaskId(), tPlan.getTaskIndex()));
        }
        containersToTasks.put(c.getContainerId(), instances);
    }
    Map<String, List<JobMasterAPI.WorkerInfo>> containersPerNode = new TreeMap<>();
    for (JobMasterAPI.WorkerInfo workerInfo : workerInfoList) {
        String name = Integer.toString(workerInfo.getWorkerID());
        List<JobMasterAPI.WorkerInfo> containerList;
        if (!containersPerNode.containsKey(name)) {
            containerList = new ArrayList<>();
            containersPerNode.put(name, containerList);
        } else {
            containerList = containersPerNode.get(name);
        }
        containerList.add(workerInfo);
    }
    Map<String, Set<Integer>> nodeToTasks = new HashMap<>();
    int i = 0;
    // we take each container as an executor
    for (Map.Entry<String, List<JobMasterAPI.WorkerInfo>> entry : containersPerNode.entrySet()) {
        Set<Integer> executorsOfGroup = new HashSet<>();
        for (JobMasterAPI.WorkerInfo workerInfo : entry.getValue()) {
            executorsOfGroup.add(workerInfo.getWorkerID());
            Set<Integer> tasksInNode = nodeToTasks.computeIfAbsent(workerInfo.getNodeInfo().getNodeIP(), k -> new HashSet<>());
            tasksInNode.addAll(containersToTasks.getOrDefault(workerInfo.getWorkerID(), Collections.emptySet()));
        }
        groupsToTasks.put(i, executorsOfGroup);
        i++;
    }
    return new LogicalPlan(containersToTasks, groupsToTasks, nodeToTasks, workerID);
}

Example 18 with WorkerSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method testUniqueSchedules4.

@Test
public void testUniqueSchedules4() {
    int parallel = 16;
    int workers = 2;
    ComputeGraph graph = createGraphWithComputeTaskAndConstraints(parallel);
    RoundRobinTaskScheduler scheduler = new RoundRobinTaskScheduler();
    scheduler.initialize(Config.newBuilder().build());
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Map<Integer, WorkerSchedulePlan> containersMap = plan1.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        Assert.assertEquals(containerPlanTaskInstances.size(), Integer.parseInt(graph.getGraphConstraints().get(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER)));
    }
}

Example 19 with WorkerSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method schedule.

/**
 * This is the base method for the data locality aware task scheduling for scheduling the batch
 * task instances. It retrieves the task vertex set of the task graph and send the set to the
 * data locality aware scheduling algorithm to allocate the batch task instances which are closer
 * to the data nodes.
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    LinkedHashMap<Integer, WorkerSchedulePlan> containerPlans = new LinkedHashMap<>();
    for (int i = 0; i < workerPlan.getNumberOfWorkers(); i++) {
        dataLocalityAwareAllocation.put(i, new ArrayList<>());
    }
    LinkedHashSet<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TaskVertexParser taskVertexParser = new TaskVertexParser();
    List<Set<Vertex>> taskVertexList = taskVertexParser.parseVertexSet(graph);
    for (Set<Vertex> vertexSet : taskVertexList) {
        Map<Integer, List<TaskInstanceId>> containerInstanceMap;
        if (vertexSet.size() > 1) {
            containerInstanceMap = dataLocalityBatchSchedulingAlgorithm(graph, vertexSet, workerPlan);
        } else {
            Vertex vertex = vertexSet.iterator().next();
            containerInstanceMap = dataLocalityBatchSchedulingAlgorithm(graph, vertex, workerPlan);
        }
        TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
        Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(containerInstanceMap, taskVertexSet);
        for (int containerId : containerInstanceMap.keySet()) {
            double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
            double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
            double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
            List<TaskInstanceId> taskTaskInstanceIds = containerInstanceMap.get(containerId);
            Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
            for (TaskInstanceId id : taskTaskInstanceIds) {
                double instanceRAMValue = instancesRamMap.get(containerId).get(id);
                double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
                double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
                Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
                taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
                containerRAMValue += instanceRAMValue;
                containerDiskValue += instanceDiskValue;
                containerCpuValue += instanceDiskValue;
            }
            Worker worker = workerPlan.getWorker(containerId);
            Resource containerResource;
            if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
                containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            } else {
                containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            }
            WorkerSchedulePlan taskWorkerSchedulePlan;
            if (containerPlans.containsKey(containerId)) {
                taskWorkerSchedulePlan = containerPlans.get(containerId);
                taskWorkerSchedulePlan.getTaskInstances().addAll(taskInstancePlanMap.values());
            } else {
                taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
                containerPlans.put(containerId, taskWorkerSchedulePlan);
            }
        }
    }
    TaskSchedulePlan taskSchedulePlan = new TaskSchedulePlan(0, new HashSet<>(containerPlans.values()));
    Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        Integer integer = entry.getKey();
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        LOG.fine("Task Details for Container Id:" + integer + "\tsize:" + containerPlanTaskInstances.size());
        for (TaskInstancePlan ip : containerPlanTaskInstances) {
            LOG.fine("TaskId:" + ip.getTaskId() + "\tTask Index" + ip.getTaskIndex() + "\tTask Name:" + ip.getTaskName());
        }
    }
    return taskSchedulePlan;
}

Example 20 with WorkerSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan in project twister2 by DSC-SPIDAL.

the class TaskScheduler method generateTaskSchedulePlan.

private TaskSchedulePlan generateTaskSchedulePlan(String className) {
    Class<?> taskSchedulerClass;
    Method method;
    TaskSchedulePlan taskSchedulePlan;
    try {
        taskSchedulerClass = getClass().getClassLoader().loadClass(className);
        Object newInstance = taskSchedulerClass.newInstance();
        method = taskSchedulerClass.getMethod("initialize", new Class<?>[] { Config.class });
        method.invoke(newInstance, config);
        method = taskSchedulerClass.getMethod("schedule", new Class<?>[] { ComputeGraph.class, WorkerPlan.class });
        taskSchedulePlan = (TaskSchedulePlan) method.invoke(newInstance, computeGraph, workerPlan);
    } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException | InstantiationException | ClassNotFoundException | TaskSchedulerException e) {
        throw new Twister2RuntimeException(e);
    }
    if (taskSchedulePlan != null) {
        Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            Integer integer = entry.getKey();
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            LOG.fine("Task Details for Container Id:" + integer);
            for (TaskInstancePlan ip : containerPlanTaskInstances) {
                LOG.fine("Task Id:" + ip.getTaskId() + "\tTask Index" + ip.getTaskIndex() + "\tTask Name:" + ip.getTaskName());
            }
        }
    }
    return taskSchedulePlan;
}