Examples with TaskAttributes edu.iu.dsc.tws.tsched.utils.TaskAttributes used on opensource projects

Example 1 with TaskAttributes

use of edu.iu.dsc.tws.tsched.utils.TaskAttributes in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method dataLocalityStreamingSchedulingAlgorithm.

/**
 * This method is primarily responsible for generating the container and task instance map which
 * is based on the task graph, its configuration, and the allocated worker plan.
 */
private Map<Integer, List<TaskInstanceId>> dataLocalityStreamingSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers, WorkerPlan workerPlan) {
    TaskAttributes taskAttributes = new TaskAttributes();
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    // Maximum task instances can be accommodated to the container
    int instancesPerContainer;
    if (!graph.getGraphConstraints().isEmpty()) {
        instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
    } else {
        instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(this.config);
    }
    // Total container capacity
    int containerCapacity = instancesPerContainer * numberOfContainers;
    int localIndex = 0;
    int containerIndex = 0;
    int totalInstances;
    // Total task instances in the taskgraph
    if (!graph.getNodeConstraints().isEmpty()) {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet, graph.getNodeConstraints());
    } else {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet);
    }
    // Map to hold the allocation of task instances into the containers/workers
    Map<Integer, List<TaskInstanceId>> dataAwareAllocationMap = new HashMap<>();
    // To check the containers can hold all the parallel task instances.
    if (containerCapacity >= totalInstances) {
        LOG.info("Task scheduling could be performed for the container capacity of " + containerCapacity + " and " + totalInstances + " task instances");
        for (int i = 0; i < numberOfContainers; i++) {
            dataAwareAllocationMap.put(i, new ArrayList<>());
        }
    } else {
        throw new TaskSchedulerException("Task scheduling couldn't be performed for the container " + "capacity of " + containerCapacity + " and " + totalInstances + " task instances");
    }
    // Parallel Task Map for the complete task graph
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    Map<String, Integer> parallelTaskMap;
    if (!graph.getNodeConstraints().isEmpty()) {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet, graph.getNodeConstraints());
    } else {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet);
    }
    /*This loop allocate the task instances to the respective container, before allocation
    it will check whether the container has reached maximum task instance size */
    for (Map.Entry<String, Integer> aTaskEntrySet : parallelTaskMap.entrySet()) {
        for (Vertex vertex : taskVertexSet) {
            if (aTaskEntrySet.getKey().equals(vertex.getName())) {
                int totalTaskInstances = vertex.getParallelism();
                int maxContainerTaskObjectSize = 0;
                List<DataTransferTimeCalculator> calList = dTTimecalculatorList(localIndex, workerPlan, dataAwareAllocationMap, containerIndex, instancesPerContainer);
                for (int i = 0; i < totalTaskInstances; i++) {
                    containerIndex = Integer.parseInt(Collections.min(calList).getNodeName().trim());
                    if (maxContainerTaskObjectSize < instancesPerContainer) {
                        dataAwareAllocationMap.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                        ++maxContainerTaskObjectSize;
                    } else {
                        throw new TaskSchedulerException("Task Scheduling couldn't be possible for the " + "present configuration, please check the number of workers, " + "maximum instances per worker");
                    }
                }
                globalTaskIndex++;
                localIndex++;
            }
        }
    }
    return dataAwareAllocationMap;
}

Example 2 with TaskAttributes

use of edu.iu.dsc.tws.tsched.utils.TaskAttributes in project twister2 by DSC-SPIDAL.

the class UserDefinedTaskScheduler method userDefinedSchedulingAlgorithm.

/**
 * This method retrieves the parallel task map and the total number of task instances for the task
 * vertex set. Then, it will allocate the instances into the number of containers allocated for
 * the task in a round robin fashion.
 *
 * The user could write their own type of allocations into the available workers using their
 * own scheduling algorithm.
 */
private static Map<Integer, List<TaskInstanceId>> userDefinedSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) {
    Map<Integer, List<TaskInstanceId>> userDefinedAllocation = new LinkedHashMap<>();
    for (int i = 0; i < numberOfContainers; i++) {
        userDefinedAllocation.put(i, new ArrayList<>());
    }
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    TaskAttributes taskAttributes = new TaskAttributes();
    int globalTaskIndex = 0;
    for (Vertex vertex : taskVertexSet) {
        int totalTaskInstances;
        if (!graph.getNodeConstraints().isEmpty()) {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex, graph.getNodeConstraints());
        } else {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex);
        }
        if (!graph.getNodeConstraints().isEmpty()) {
            int instancesPerWorker = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
            int maxTaskInstancesPerContainer = 0;
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                if (maxTaskInstancesPerContainer < instancesPerWorker) {
                    userDefinedAllocation.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                    ++maxTaskInstancesPerContainer;
                } else {
                    throw new TaskSchedulerException("Task Scheduling couldn't be possible for the present" + "configuration, please check the number of workers, " + "maximum instances per worker");
                }
            }
        } else {
            String task = vertex.getName();
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                userDefinedAllocation.get(containerIndex).add(new TaskInstanceId(task, globalTaskIndex, i));
            }
        }
        globalTaskIndex++;
    }
    return userDefinedAllocation;
}

Example 3 with TaskAttributes

use of edu.iu.dsc.tws.tsched.utils.TaskAttributes in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method initialize.

/**
 * This method initialize the task instance values with the values specified in the task config
 * object.
 */
@Override
public void initialize(Config cfg) {
    this.config = cfg;
    this.instanceRAM = TaskSchedulerContext.taskInstanceRam(config);
    this.instanceDisk = TaskSchedulerContext.taskInstanceDisk(config);
    this.instanceCPU = TaskSchedulerContext.taskInstanceCpu(config);
    this.batchTaskAllocation = new LinkedHashMap<>();
    this.taskAttributes = new TaskAttributes();
}

Example 4 with TaskAttributes

use of edu.iu.dsc.tws.tsched.utils.TaskAttributes in project twister2 by DSC-SPIDAL.

the class RoundRobinBatchTaskScheduler method initialize.

/**
 * This method initialize the task instance values with the values specified in the task config
 * object.
 */
@Override
public void initialize(Config cfg) {
    this.config = cfg;
    this.instanceRAM = TaskSchedulerContext.taskInstanceRam(config);
    this.instanceDisk = TaskSchedulerContext.taskInstanceDisk(config);
    this.instanceCPU = TaskSchedulerContext.taskInstanceCpu(config);
    this.roundRobinAllocation = new HashMap<>();
    this.taskAttributes = new TaskAttributes();
}

Example 5 with TaskAttributes

use of edu.iu.dsc.tws.tsched.utils.TaskAttributes in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method initialize.

/**
 * This method first initialize the task instance values with default task instance ram, disk, and
 * cpu values from the task scheduler context.
 */
@Override
public void initialize(Config cfg) {
    this.config = cfg;
    this.instanceRAM = TaskSchedulerContext.taskInstanceRam(this.config);
    this.instanceDisk = TaskSchedulerContext.taskInstanceDisk(this.config);
    this.instanceCPU = TaskSchedulerContext.taskInstanceCpu(this.config);
    this.dataNodeLocatorUtils = new DataNodeLocatorUtils(config);
    this.dataLocalityAwareAllocation = new HashMap<>();
    this.taskAttributes = new TaskAttributes();
}