Examples with TaskInstanceId edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId used on opensource projects

Example 1 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class RoundRobinBatchTaskScheduler method nonAttributeBasedAllocation.

private Map<Integer, List<TaskInstanceId>> nonAttributeBasedAllocation(Map<String, Integer> parallelTaskMap) {
    int containerIndex = 0;
    for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
        String task = e.getKey();
        int numberOfInstances = e.getValue();
        for (int taskIndex = 0; taskIndex < numberOfInstances; taskIndex++) {
            roundRobinAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
            ++containerIndex;
            if (containerIndex >= roundRobinAllocation.size()) {
                containerIndex = 0;
            }
        }
        gTaskId++;
    }
    return roundRobinAllocation;
}

Example 2 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class RoundRobinBatchTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph computeGraph, WorkerPlan workerPlan) {
    Map<Integer, List<TaskInstanceId>> containerInstanceMap;
    Map<Integer, WorkerSchedulePlan> containerPlans = new LinkedHashMap<>();
    for (int i = 0; i < workerPlan.getNumberOfWorkers(); i++) {
        roundRobinAllocation.put(i, new ArrayList<>());
    }
    // To retrieve the batch task instances(it may be single task vertex or a batch of task vertices)
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(computeGraph.getTaskVertexSet());
    TaskVertexParser taskGraphParser = new TaskVertexParser();
    List<Set<Vertex>> taskVertexList = taskGraphParser.parseVertexSet(computeGraph);
    for (Set<Vertex> vertexSet : taskVertexList) {
        if (vertexSet.size() > 1) {
            containerInstanceMap = roundRobinBatchSchedulingAlgorithm(computeGraph, vertexSet);
        } else {
            Vertex vertex = vertexSet.iterator().next();
            containerInstanceMap = roundRobinBatchSchedulingAlgorithm(computeGraph, vertex);
        }
        TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
        Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(containerInstanceMap, taskVertexSet);
        for (int containerId : containerInstanceMap.keySet()) {
            double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
            double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
            double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
            List<TaskInstanceId> taskTaskInstanceIds = containerInstanceMap.get(containerId);
            Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
            for (TaskInstanceId id : taskTaskInstanceIds) {
                double instanceRAMValue = instancesRamMap.get(containerId).get(id);
                double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
                double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
                Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
                taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
                containerRAMValue += instanceRAMValue;
                containerDiskValue += instanceDiskValue;
                containerCpuValue += instanceDiskValue;
            }
            Worker worker = workerPlan.getWorker(containerId);
            Resource containerResource;
            if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
                containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            } else {
                containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            }
            WorkerSchedulePlan taskWorkerSchedulePlan;
            if (containerPlans.containsKey(containerId)) {
                taskWorkerSchedulePlan = containerPlans.get(containerId);
                taskWorkerSchedulePlan.getTaskInstances().addAll(taskInstancePlanMap.values());
            } else {
                taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
                containerPlans.put(containerId, taskWorkerSchedulePlan);
            }
        }
    }
    return new TaskSchedulePlan(0, new HashSet<>(containerPlans.values()));
}

Example 3 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method nonAttributeBasedAllocation.

private Map<Integer, List<TaskInstanceId>> nonAttributeBasedAllocation(Map<String, Integer> parallelTaskMap, WorkerPlan workerPlan) {
    List<DataTransferTimeCalculator> workerNodeList = getWorkerNodeList(workerPlan);
    int instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(config);
    int containerIndex = Integer.parseInt(workerNodeList.get(0).getNodeName());
    for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
        String task = e.getKey();
        int taskParallelism = e.getValue();
        for (int taskIndex = 0, maxTaskObject = 0; taskIndex < taskParallelism; taskIndex++) {
            dataLocalityAwareAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
            maxTaskObject++;
            if (maxTaskObject == instancesPerContainer) {
                ++containerIndex;
            }
        }
        containerIndex = 0;
        gTaskId++;
    }
    return dataLocalityAwareAllocation;
}

Example 4 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method attributeBasedAllocation.

private Map<Integer, List<TaskInstanceId>> attributeBasedAllocation(Map<String, Integer> parallelTaskMap, ComputeGraph graph, WorkerPlan workerPlan) {
    List<DataTransferTimeCalculator> workerNodeList = getWorkerNodeList(workerPlan);
    int containerIndex = Integer.parseInt(workerNodeList.get(0).getNodeName());
    int instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
    for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
        String task = e.getKey();
        int taskParallelism = e.getValue();
        for (int taskIndex = 0, maxTaskObject = 0; taskIndex < taskParallelism; taskIndex++) {
            dataLocalityAwareAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
            maxTaskObject++;
            if (maxTaskObject == instancesPerContainer) {
                ++containerIndex;
            }
        }
        containerIndex = 0;
        gTaskId++;
    }
    return dataLocalityAwareAllocation;
}

Example 5 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method dataLocalityStreamingSchedulingAlgorithm.

/**
 * This method is primarily responsible for generating the container and task instance map which
 * is based on the task graph, its configuration, and the allocated worker plan.
 */
private Map<Integer, List<TaskInstanceId>> dataLocalityStreamingSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers, WorkerPlan workerPlan) {
    TaskAttributes taskAttributes = new TaskAttributes();
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    // Maximum task instances can be accommodated to the container
    int instancesPerContainer;
    if (!graph.getGraphConstraints().isEmpty()) {
        instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
    } else {
        instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(this.config);
    }
    // Total container capacity
    int containerCapacity = instancesPerContainer * numberOfContainers;
    int localIndex = 0;
    int containerIndex = 0;
    int totalInstances;
    // Total task instances in the taskgraph
    if (!graph.getNodeConstraints().isEmpty()) {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet, graph.getNodeConstraints());
    } else {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet);
    }
    // Map to hold the allocation of task instances into the containers/workers
    Map<Integer, List<TaskInstanceId>> dataAwareAllocationMap = new HashMap<>();
    // To check the containers can hold all the parallel task instances.
    if (containerCapacity >= totalInstances) {
        LOG.info("Task scheduling could be performed for the container capacity of " + containerCapacity + " and " + totalInstances + " task instances");
        for (int i = 0; i < numberOfContainers; i++) {
            dataAwareAllocationMap.put(i, new ArrayList<>());
        }
    } else {
        throw new TaskSchedulerException("Task scheduling couldn't be performed for the container " + "capacity of " + containerCapacity + " and " + totalInstances + " task instances");
    }
    // Parallel Task Map for the complete task graph
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    Map<String, Integer> parallelTaskMap;
    if (!graph.getNodeConstraints().isEmpty()) {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet, graph.getNodeConstraints());
    } else {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet);
    }
    /*This loop allocate the task instances to the respective container, before allocation
    it will check whether the container has reached maximum task instance size */
    for (Map.Entry<String, Integer> aTaskEntrySet : parallelTaskMap.entrySet()) {
        for (Vertex vertex : taskVertexSet) {
            if (aTaskEntrySet.getKey().equals(vertex.getName())) {
                int totalTaskInstances = vertex.getParallelism();
                int maxContainerTaskObjectSize = 0;
                List<DataTransferTimeCalculator> calList = dTTimecalculatorList(localIndex, workerPlan, dataAwareAllocationMap, containerIndex, instancesPerContainer);
                for (int i = 0; i < totalTaskInstances; i++) {
                    containerIndex = Integer.parseInt(Collections.min(calList).getNodeName().trim());
                    if (maxContainerTaskObjectSize < instancesPerContainer) {
                        dataAwareAllocationMap.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                        ++maxContainerTaskObjectSize;
                    } else {
                        throw new TaskSchedulerException("Task Scheduling couldn't be possible for the " + "present configuration, please check the number of workers, " + "maximum instances per worker");
                    }
                }
                globalTaskIndex++;
                localIndex++;
            }
        }
    }
    return dataAwareAllocationMap;
}