Examples with TaskSchedulePlan edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan used on opensource projects

Example 31 with TaskSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method testUniqueSchedules4.

@Test
public void testUniqueSchedules4() {
    int parallel = 4;
    int workers = 2;
    ComputeGraph graph = createGraphWithMultipleComputeTaskAndConstraints(parallel);
    DataLocalityBatchTaskScheduler scheduler = new DataLocalityBatchTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config, 1);
    generateData(config);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Assert.assertNotNull(plan1);
    Map<Integer, WorkerSchedulePlan> containersMap = plan1.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        Assert.assertEquals(containerPlanTaskInstances.size(), workers * graph.getTaskVertexSet().size());
    }
}

Example 32 with TaskSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method testUniqueSchedules3.

@Test
public void testUniqueSchedules3() {
    int parallel = 4;
    int workers = 2;
    ComputeGraph graph = createGraphWithComputeTaskAndConstraints(parallel);
    DataLocalityBatchTaskScheduler scheduler = new DataLocalityBatchTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config, 1);
    generateData(config);
    DataGenerator dataGenerator = new DataGenerator(config);
    dataGenerator.generate(new Path(String.valueOf(config.get(DataObjectConstants.DINPUT_DIRECTORY))), 1000, 2);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Assert.assertNotNull(plan1);
    Map<Integer, WorkerSchedulePlan> containersMap = plan1.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        LOG.info("container plan instances and parallel:" + containerPlanTaskInstances.size() + "\t" + workers * graph.getTaskVertexSet().size());
        Assert.assertEquals(containerPlanTaskInstances.size(), workers * graph.getTaskVertexSet().size());
    }
}

Example 33 with TaskSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan in project twister2 by DSC-SPIDAL.

the class DataLocalityTaskSchedulerTest method testUniqueSchedules2.

@Test
public void testUniqueSchedules2() {
    int parallel = 10;
    int workers = 2;
    ComputeGraph graph = createGraphWithConstraints(parallel);
    DataLocalityStreamingTaskScheduler scheduler = new DataLocalityStreamingTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config, 1);
    generateData(config);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Assert.assertNotNull(plan1);
    Map<Integer, WorkerSchedulePlan> containersMap = plan1.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        Assert.assertEquals(containerPlanTaskInstances.size(), Integer.parseInt(graph.getGraphConstraints().get(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER)));
    }
}

Example 34 with TaskSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 *
 * @return TaskSchedulePlan
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph computeGraph, WorkerPlan workerPlan) {
    // Allocate the task instances into the containers/workers
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    // To get the vertex set from the taskgraph
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(computeGraph.getTaskVertexSet());
    // Allocate the task instances into the logical containers.
    Map<Integer, List<TaskInstanceId>> roundRobinContainerInstanceMap = roundRobinSchedulingAlgorithm(computeGraph, workerPlan.getNumberOfWorkers());
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(roundRobinContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(roundRobinContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(roundRobinContainerInstanceMap, taskVertexSet);
    for (int containerId : roundRobinContainerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = roundRobinContainerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        // Create the container resource value based on the worker plan
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
        }
        // Schedule the task instance plan into the task container plan.
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new LinkedHashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
    }
    return new TaskSchedulePlan(0, workerSchedulePlans);
}

Example 35 with TaskSchedulePlan

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan in project twister2 by DSC-SPIDAL.

the class TaskSchedulerTest method testUniqueSchedules2.

@Test
public void testUniqueSchedules2() {
    int parallel = 2;
    ComputeGraph graph = createBatchGraph(parallel);
    TaskScheduler scheduler = new TaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config);
    WorkerPlan workerPlan = createWorkPlan(parallel);
    if (graph.getOperationMode().equals("BATCH")) {
        Assert.assertEquals(scheduler.getClass(), TaskSchedulerContext.batchTaskSchedulingClass(config));
    }
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Assert.assertNotNull(plan1);
}