Examples with TaskInstanceId edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId used on opensource projects

Example 6 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method schedule.

/**
 * This is the base method for the data locality aware task scheduling for scheduling the
 * streaming task instances. It retrieves the task vertex set of the task graph and send the set
 * to the data locality aware scheduling algorithm to schedule the streaming task instances
 * which are closer to the data nodes.
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    // Represents task schedule plan Id
    int taskSchedulePlanId = 0;
    Set<WorkerSchedulePlan> workerSchedulePlans = new HashSet<>();
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    Map<Integer, List<TaskInstanceId>> containerInstanceMap = dataLocalityStreamingSchedulingAlgorithm(graph, workerPlan.getNumberOfWorkers(), workerPlan);
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(containerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(containerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(containerInstanceMap, taskVertexSet);
    for (int containerId : containerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = containerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
        }
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
    }
    return new TaskSchedulePlan(taskSchedulePlanId, workerSchedulePlans);
}

Example 7 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class UserDefinedTaskScheduler method userDefinedSchedulingAlgorithm.

/**
 * This method retrieves the parallel task map and the total number of task instances for the task
 * vertex set. Then, it will allocate the instances into the number of containers allocated for
 * the task in a round robin fashion.
 *
 * The user could write their own type of allocations into the available workers using their
 * own scheduling algorithm.
 */
private static Map<Integer, List<TaskInstanceId>> userDefinedSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) {
    Map<Integer, List<TaskInstanceId>> userDefinedAllocation = new LinkedHashMap<>();
    for (int i = 0; i < numberOfContainers; i++) {
        userDefinedAllocation.put(i, new ArrayList<>());
    }
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    TaskAttributes taskAttributes = new TaskAttributes();
    int globalTaskIndex = 0;
    for (Vertex vertex : taskVertexSet) {
        int totalTaskInstances;
        if (!graph.getNodeConstraints().isEmpty()) {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex, graph.getNodeConstraints());
        } else {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex);
        }
        if (!graph.getNodeConstraints().isEmpty()) {
            int instancesPerWorker = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
            int maxTaskInstancesPerContainer = 0;
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                if (maxTaskInstancesPerContainer < instancesPerWorker) {
                    userDefinedAllocation.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                    ++maxTaskInstancesPerContainer;
                } else {
                    throw new TaskSchedulerException("Task Scheduling couldn't be possible for the present" + "configuration, please check the number of workers, " + "maximum instances per worker");
                }
            }
        } else {
            String task = vertex.getName();
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                userDefinedAllocation.get(containerIndex).add(new TaskInstanceId(task, globalTaskIndex, i));
            }
        }
        globalTaskIndex++;
    }
    return userDefinedAllocation;
}

Example 8 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class UserDefinedTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 *
 * @return TaskSchedulePlan
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    int taskSchedulePlanId = 0;
    // Allocate the task instances into the containers/workers
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    // To get the vertex set from the taskgraph
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    // Allocate the task instances into the logical containers.
    Map<Integer, List<TaskInstanceId>> userDefinedContainerInstanceMap = userDefinedSchedulingAlgorithm(graph, workerPlan.getNumberOfWorkers());
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    for (int containerId : userDefinedContainerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = userDefinedContainerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        // Create the container resource value based on the worker plan
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            LOG.fine("Worker (if loop):" + containerId + "\tRam:" + worker.getRam() + "\tDisk:" + worker.getDisk() + "\tCpu:" + worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            LOG.fine("Worker (else loop):" + containerId + "\tRam:" + containerRAMValue + "\tDisk:" + containerDiskValue + "\tCpu:" + containerCpuValue);
        }
        // Schedule the task instance plan into the task container plan.
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
    }
    return new TaskSchedulePlan(taskSchedulePlanId, workerSchedulePlans);
}

Example 9 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method addInstance.

public TaskSchedulePlanBuilder addInstance(Integer containerId, String taskName) throws TaskSchedulerException {
    initContainer(containerId);
    int taskId = taskIds.isEmpty() ? 1 : taskIds.last() + 1;
    int taskIndex = taskIndexes.get(taskName) != null ? taskIndexes.get(taskName).last() + 1 : 0;
    TaskInstanceId taskInstanceId = new TaskInstanceId(taskName, taskId, taskIndex);
    Resource resource = TaskScheduleUtils.getResourceRequirement(taskName, this.taskRamMap, this.instanceDefaultResourceValue, this.containerMaximumResourceValue, this.requestedContainerPadding);
    try {
        addToContainer(containers.get(containerId), new TaskInstancePlan(taskInstanceId, resource), taskIndexes, taskIds);
    } catch (TaskSchedulerException e) {
        throw new TaskSchedulerException(String.format("Insufficient container resources to add instance %s with resources %s to container %d.", taskInstanceId, resource, containerId), e);
    }
    LOG.info("Task id, index, name:" + taskId + "\t" + taskIndex + "\t" + taskName + "\tadded to Container:" + containers.get(containerId));
    return this;
}

Example 10 with TaskInstanceId

use of edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method buildContainerPlans.

private Set<WorkerSchedulePlan> buildContainerPlans(Map<Integer, Container> containerValue, Map<String, Double> taskramMap, Resource instdefaultresourcevalue) {
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    try {
        for (Integer containerId : containerValue.keySet()) {
            Container container = containerValue.get(containerId);
            if (container.getTaskInstances().size() == 0) {
                continue;
            }
            double containerRAMValue = 0.0;
            double containerDiskValue = 0.0;
            double containerCPUValue = 0.0;
            Set<TaskInstancePlan> taskInstancePlans = new HashSet<>();
            for (TaskInstancePlan taskInstancePlan : container.getTaskInstances()) {
                TaskInstanceId taskInstanceId = new TaskInstanceId(taskInstancePlan.getTaskName(), taskInstancePlan.getTaskId(), taskInstancePlan.getTaskIndex());
                double instanceRAMValue;
                if (taskramMap.containsKey(taskInstanceId.getTaskName())) {
                    instanceRAMValue = taskramMap.get(taskInstanceId.getTaskName());
                } else {
                    instanceRAMValue = instdefaultresourcevalue.getRam();
                }
                containerRAMValue += instanceRAMValue;
                double instanceDiskValue = instdefaultresourcevalue.getDisk();
                containerDiskValue += instanceDiskValue;
                double instanceCPUValue = instdefaultresourcevalue.getCpu();
                containerCPUValue += instanceCPUValue;
                LOG.fine("Resource Container Values:" + "Ram Value:" + containerRAMValue + "\t" + "Cpu Value:" + containerCPUValue + "\t" + "Disk Value:" + containerDiskValue);
                Resource resource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
                taskInstancePlans.add(new TaskInstancePlan(taskInstanceId.getTaskName(), taskInstanceId.getTaskId(), taskInstanceId.getTaskIndex(), resource));
            }
            containerCPUValue += (requestedContainerPadding * containerCPUValue) / 100;
            containerRAMValue += containerRAMValue + requestedContainerPadding;
            containerDiskValue += containerDiskValue + requestedContainerPadding;
            Resource resource = new Resource(containerRAMValue, containerDiskValue, containerCPUValue);
            WorkerSchedulePlan workerSchedulePlan = new WorkerSchedulePlan(containerId, taskInstancePlans, resource);
            workerSchedulePlans.add(workerSchedulePlan);
        }
    } catch (TaskSchedulerException ne) {
        throw new RuntimeException("Exception Occured" + ne.getMessage());
    }
    return workerSchedulePlans;
}