Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 11 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getParallelTaskMap.

/**
 * This method is to generate the parallel task map for the task vertex. If the user specifies the
 * parallelism value greater than or equal "1" will be considered as a parallelism value.
 * Otherwise, the system assign the default parallelism value to the task vertex from the task
 * scheduling configuration file.
 */
public Map<String, Integer> getParallelTaskMap(Vertex taskVertex, Map<String, Map<String, String>> nodeConstraintsMap) {
    Map<String, Integer> parallelTaskMap = new LinkedHashMap<>();
    Config config = taskVertex.getConfig();
    String taskName = taskVertex.getName();
    int parallelTaskCount = 0;
    if (nodeConstraintsMap.get(taskName) != null) {
        Map<String, String> vertexConstraintMap = nodeConstraintsMap.get(taskName);
        if (vertexConstraintMap.containsKey(Context.TWISTER2_TASK_INSTANCE_ODD_PARALLELISM)) {
            parallelTaskCount = Integer.valueOf(String.valueOf(vertexConstraintMap.get(Context.TWISTER2_TASK_INSTANCE_ODD_PARALLELISM)));
        } else {
            parallelTaskCount = TaskSchedulerContext.taskParallelism(config);
        }
    }
    parallelTaskMap.put(taskName, parallelTaskCount);
    return parallelTaskMap;
}

Example 12 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getTaskNetworkMap.

/**
 * This method retrieve the set of task vertices and check if the task vertex has the user
 * specified network value. If the user doesn't specify the required network configuration it will
 * assign the default network value from the task configuration file and store it in the map.
 */
public Map<String, Double> getTaskNetworkMap(Set<Vertex> taskVertices) {
    Map<String, Double> taskNetworkMap = new LinkedHashMap<>();
    Object network;
    double requiredNetwork;
    for (Vertex task : taskVertices) {
        Config config = task.getConfig();
        if (config.get("Network") != null) {
            network = config.get("Network");
            requiredNetwork = (double) ((Integer) network);
        } else {
            requiredNetwork = TaskSchedulerContext.taskInstanceNetwork(config);
        }
        taskNetworkMap.put(task.getName(), requiredNetwork);
    }
    return taskNetworkMap;
}

Example 13 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getTaskCPUMap.

/**
 * This method retrieve the set of task vertices and check if the task vertex has the user
 * specified cpu value. If the user doesn't specify the required cpu configuration it will assign
 * the default cpu value from the task configuration file and store it in the map.
 */
public Map<String, Double> getTaskCPUMap(Set<Vertex> taskVertices) {
    Map<String, Double> taskCPUMap = new LinkedHashMap<>();
    Object cpu;
    double requiredCpu;
    for (Vertex task : taskVertices) {
        Config config = task.getConfig();
        if (config.get("Cpu") != null) {
            cpu = config.get("Cpu");
            requiredCpu = (double) ((Integer) cpu);
        } else {
            requiredCpu = TaskSchedulerContext.taskInstanceCpu(config);
        }
        taskCPUMap.put(task.getName(), requiredCpu);
    }
    return taskCPUMap;
}

Example 14 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method testUniqueSchedules1.

@Test
public void testUniqueSchedules1() {
    int parallel = 4;
    int workers = 2;
    ComputeGraph graph = createGraph(parallel);
    DataLocalityBatchTaskScheduler scheduler = new DataLocalityBatchTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config);
    generateData(config);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    WorkerPlan workerPlan2 = createWorkPlan2(workers);
    for (int i = 0; i < 10; i++) {
        TaskSchedulePlan plan2 = scheduler.schedule(graph, workerPlan2);
        Assert.assertEquals(plan1.getContainers().size(), plan2.getContainers().size());
        Map<Integer, WorkerSchedulePlan> containersMap = plan2.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            Assert.assertEquals(containerPlanTaskInstances.size() / graph.getTaskVertexSet().size(), TaskSchedulerContext.defaultTaskInstancesPerContainer(config));
        }
    }
}

Example 15 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class DataLocalityTaskSchedulerTest method testUniqueSchedules1.

@Test
public void testUniqueSchedules1() {
    int parallel = 2;
    int workers = 2;
    ComputeGraph graph = createGraph(parallel);
    DataLocalityStreamingTaskScheduler scheduler = new DataLocalityStreamingTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config, 1);
    generateData(config);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    WorkerPlan workerPlan2 = createWorkPlan2(workers);
    for (int i = 0; i < 10; i++) {
        TaskSchedulePlan plan2 = scheduler.schedule(graph, workerPlan2);
        Assert.assertEquals(plan1.getContainers().size(), plan2.getContainers().size());
        Map<Integer, WorkerSchedulePlan> containersMap = plan2.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            Assert.assertEquals(containerPlanTaskInstances.size(), TaskSchedulerContext.defaultTaskInstancesPerContainer(config));
        }
    }
}