Examples with TaskInstanceId edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId used on opensource projects

Example 16 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method independentTaskWorkerAllocation.

/**
 * This method is to allocate the task for the individual task graph.
 *
 * @param graph
 * @param vertex
 * @param numberOfContainers
 * @param globalTaskIndex
 */
private void independentTaskWorkerAllocation(ComputeGraph graph, Vertex vertex, int numberOfContainers, int globalTaskIndex) {
    int totalTaskInstances;
    if (!graph.getNodeConstraints().isEmpty()) {
        totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex, graph.getNodeConstraints());
    } else {
        totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex);
    }
    if (!graph.getNodeConstraints().isEmpty()) {
        int instancesPerWorker = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
        int maxTaskInstancesPerContainer = 0;
        int containerIndex;
        for (int i = 0; i < totalTaskInstances; i++) {
            containerIndex = i % numberOfContainers;
            if (maxTaskInstancesPerContainer < instancesPerWorker) {
                batchTaskAllocation.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                ++maxTaskInstancesPerContainer;
            } else {
                throw new TaskSchedulerException("Task Scheduling couldn't be possible for the present" + "configuration, please check the number of workers, " + "maximum instances per worker");
            }
        }
    } else {
        String task = vertex.getName();
        int containerIndex;
        for (int i = 0; i < totalTaskInstances; i++) {
            containerIndex = i % numberOfContainers;
            batchTaskAllocation.get(containerIndex).add(new TaskInstanceId(task, globalTaskIndex, i));
        }
    }
}

Example 17 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class RoundRobinBatchTaskScheduler method attributeBasedAllocation.

private Map<Integer, List<TaskInstanceId>> attributeBasedAllocation(Map<String, Integer> parallelTaskMap, ComputeGraph graph) {
    int containerIndex = 0;
    int instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
    for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
        String task = e.getKey();
        int taskParallelism = e.getValue();
        int numberOfInstances;
        if (instancesPerContainer < taskParallelism) {
            numberOfInstances = taskParallelism;
        } else {
            numberOfInstances = instancesPerContainer;
        }
        for (int taskIndex = 0; taskIndex < numberOfInstances; taskIndex++) {
            roundRobinAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
            ++containerIndex;
            if (containerIndex >= roundRobinAllocation.size()) {
                containerIndex = 0;
            }
        }
        gTaskId++;
    }
    return roundRobinAllocation;
}

Example 18 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method schedule.

/**
 * This is the base method for the data locality aware task scheduling for scheduling the batch
 * task instances. It retrieves the task vertex set of the task graph and send the set to the
 * data locality aware scheduling algorithm to allocate the batch task instances which are closer
 * to the data nodes.
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    LinkedHashMap<Integer, WorkerSchedulePlan> containerPlans = new LinkedHashMap<>();
    for (int i = 0; i < workerPlan.getNumberOfWorkers(); i++) {
        dataLocalityAwareAllocation.put(i, new ArrayList<>());
    }
    LinkedHashSet<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TaskVertexParser taskVertexParser = new TaskVertexParser();
    List<Set<Vertex>> taskVertexList = taskVertexParser.parseVertexSet(graph);
    for (Set<Vertex> vertexSet : taskVertexList) {
        Map<Integer, List<TaskInstanceId>> containerInstanceMap;
        if (vertexSet.size() > 1) {
            containerInstanceMap = dataLocalityBatchSchedulingAlgorithm(graph, vertexSet, workerPlan);
        } else {
            Vertex vertex = vertexSet.iterator().next();
            containerInstanceMap = dataLocalityBatchSchedulingAlgorithm(graph, vertex, workerPlan);
        }
        TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
        Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(containerInstanceMap, taskVertexSet);
        for (int containerId : containerInstanceMap.keySet()) {
            double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
            double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
            double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
            List<TaskInstanceId> taskTaskInstanceIds = containerInstanceMap.get(containerId);
            Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
            for (TaskInstanceId id : taskTaskInstanceIds) {
                double instanceRAMValue = instancesRamMap.get(containerId).get(id);
                double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
                double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
                Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
                taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
                containerRAMValue += instanceRAMValue;
                containerDiskValue += instanceDiskValue;
                containerCpuValue += instanceDiskValue;
            }
            Worker worker = workerPlan.getWorker(containerId);
            Resource containerResource;
            if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
                containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            } else {
                containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            }
            WorkerSchedulePlan taskWorkerSchedulePlan;
            if (containerPlans.containsKey(containerId)) {
                taskWorkerSchedulePlan = containerPlans.get(containerId);
                taskWorkerSchedulePlan.getTaskInstances().addAll(taskInstancePlanMap.values());
            } else {
                taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
                containerPlans.put(containerId, taskWorkerSchedulePlan);
            }
        }
    }
    TaskSchedulePlan taskSchedulePlan = new TaskSchedulePlan(0, new HashSet<>(containerPlans.values()));
    Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        Integer integer = entry.getKey();
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        LOG.fine("Task Details for Container Id:" + integer + "\tsize:" + containerPlanTaskInstances.size());
        for (TaskInstancePlan ip : containerPlanTaskInstances) {
            LOG.fine("TaskId:" + ip.getTaskId() + "\tTask Index" + ip.getTaskIndex() + "\tTask Name:" + ip.getTaskName());
        }
    }
    return taskSchedulePlan;
}

Example 19 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class TaskInstanceMapCalculation method getInstancesRamMapInContainer.

/**
 *  It receives the container instance allocation map and calculate the required number of
 *  task instances with ram values.
 * @param containerInstanceAllocationMap
 * @param taskVertexSet
 * @return
 */
public Map<Integer, Map<TaskInstanceId, Double>> getInstancesRamMapInContainer(Map<Integer, List<TaskInstanceId>> containerInstanceAllocationMap, Set<Vertex> taskVertexSet) {
    Map<String, Double> ramMap = taskAttributes.getTaskRamMap(taskVertexSet);
    HashMap<Integer, Map<TaskInstanceId, Double>> instancesRamContainerMap = new HashMap<>();
    for (int containerId : containerInstanceAllocationMap.keySet()) {
        Double usedRamValue = 0.0;
        List<TaskInstanceId> taskInstanceIds = containerInstanceAllocationMap.get(containerId);
        Map<TaskInstanceId, Double> containerRam = new HashMap<>();
        instancesRamContainerMap.put(containerId, containerRam);
        List<TaskInstanceId> instancesToBeCalculated = new ArrayList<>();
        for (TaskInstanceId taskInstanceId : taskInstanceIds) {
            String taskName = taskInstanceId.getTaskName();
            if (ramMap.containsKey(taskName)) {
                Double ramValue = ramMap.get(taskName);
                containerRam.put(taskInstanceId, ramValue);
                usedRamValue += ramValue;
            } else {
                instancesToBeCalculated.add(taskInstanceId);
            }
        }
        Double containerRamValue = getContainerRamValue(containerInstanceAllocationMap);
        int instancesAllocationSize = instancesToBeCalculated.size();
        if (instancesAllocationSize != 0) {
            Double instanceRequiredRam = instanceRAM;
            if (!containerRamValue.equals(NOT_SPECIFIED_NUMBER_VALUE)) {
                double remainingRam = containerRamValue - DEFAULT_RAM_PADDING_PER_CONTAINER - usedRamValue;
                instanceRequiredRam = remainingRam / instancesAllocationSize;
            }
            for (TaskInstanceId taskInstanceId : instancesToBeCalculated) {
                containerRam.put(taskInstanceId, instanceRequiredRam);
            }
            LOG.fine("Instances Required Ram:\t" + instanceRequiredRam + "\n");
        }
    }
    return instancesRamContainerMap;
}

Example 20 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskScheduler method roundRobinSchedulingAlgorithm.

/**
 * This method retrieves the parallel task map and the total number of task instances for the task
 * vertex set. Then, it will allocate the instances into the number of containers allocated for
 * the task in a round robin fashion.
 */
private Map<Integer, List<TaskInstanceId>> roundRobinSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) throws TaskSchedulerException {
    Map<Integer, List<TaskInstanceId>> roundrobinAllocation = new LinkedHashMap<>();
    for (int i = 0; i < numberOfContainers; i++) {
        roundrobinAllocation.put(i, new ArrayList<>());
    }
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    TaskAttributes taskAttributes = new TaskAttributes();
    int globalTaskIndex = 0;
    for (Vertex vertex : taskVertexSet) {
        int totalTaskInstances;
        if (!graph.getNodeConstraints().isEmpty()) {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex, graph.getNodeConstraints());
        } else {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex);
        }
        if (!graph.getNodeConstraints().isEmpty()) {
            int instancesPerWorker = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
            int maxTaskInstancesPerContainer = 0;
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                if (maxTaskInstancesPerContainer < instancesPerWorker) {
                    roundrobinAllocation.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                    ++maxTaskInstancesPerContainer;
                } else {
                    throw new TaskSchedulerException("Task Scheduling couldn't be possible for the present" + "configuration, please check the number of workers, " + "maximum instances per worker");
                }
            }
        } else {
            String task = vertex.getName();
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                roundrobinAllocation.get(containerIndex).add(new TaskInstanceId(task, globalTaskIndex, i));
            }
        }
        globalTaskIndex++;
    }
    return roundrobinAllocation;
}