Examples with Resource edu.iu.dsc.tws.api.compute.schedule.elements.Resource used on opensource projects

Example 1 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class RoundRobinBatchTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph computeGraph, WorkerPlan workerPlan) {
    Map<Integer, List<TaskInstanceId>> containerInstanceMap;
    Map<Integer, WorkerSchedulePlan> containerPlans = new LinkedHashMap<>();
    for (int i = 0; i < workerPlan.getNumberOfWorkers(); i++) {
        roundRobinAllocation.put(i, new ArrayList<>());
    }
    // To retrieve the batch task instances(it may be single task vertex or a batch of task vertices)
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(computeGraph.getTaskVertexSet());
    TaskVertexParser taskGraphParser = new TaskVertexParser();
    List<Set<Vertex>> taskVertexList = taskGraphParser.parseVertexSet(computeGraph);
    for (Set<Vertex> vertexSet : taskVertexList) {
        if (vertexSet.size() > 1) {
            containerInstanceMap = roundRobinBatchSchedulingAlgorithm(computeGraph, vertexSet);
        } else {
            Vertex vertex = vertexSet.iterator().next();
            containerInstanceMap = roundRobinBatchSchedulingAlgorithm(computeGraph, vertex);
        }
        TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
        Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(containerInstanceMap, taskVertexSet);
        for (int containerId : containerInstanceMap.keySet()) {
            double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
            double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
            double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
            List<TaskInstanceId> taskTaskInstanceIds = containerInstanceMap.get(containerId);
            Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
            for (TaskInstanceId id : taskTaskInstanceIds) {
                double instanceRAMValue = instancesRamMap.get(containerId).get(id);
                double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
                double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
                Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
                taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
                containerRAMValue += instanceRAMValue;
                containerDiskValue += instanceDiskValue;
                containerCpuValue += instanceDiskValue;
            }
            Worker worker = workerPlan.getWorker(containerId);
            Resource containerResource;
            if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
                containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            } else {
                containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            }
            WorkerSchedulePlan taskWorkerSchedulePlan;
            if (containerPlans.containsKey(containerId)) {
                taskWorkerSchedulePlan = containerPlans.get(containerId);
                taskWorkerSchedulePlan.getTaskInstances().addAll(taskInstancePlanMap.values());
            } else {
                taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
                containerPlans.put(containerId, taskWorkerSchedulePlan);
            }
        }
    }
    return new TaskSchedulePlan(0, new HashSet<>(containerPlans.values()));
}

Example 2 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class Container method getTotalUsedResources.

private Resource getTotalUsedResources() {
    double usedRam = 0.0;
    double usedCpuCores = 0.0;
    double usedDisk = 0.0;
    for (TaskInstancePlan instancePlan : this.taskInstances) {
        Resource instancePlanResource = instancePlan.getResource();
        usedRam += instancePlanResource.getRam();
        usedCpuCores += instancePlanResource.getCpu();
        usedDisk += instancePlanResource.getDisk();
    }
    return new Resource(usedRam, usedDisk, usedCpuCores);
}

Example 3 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method schedule.

/**
 * This is the base method for the data locality aware task scheduling for scheduling the
 * streaming task instances. It retrieves the task vertex set of the task graph and send the set
 * to the data locality aware scheduling algorithm to schedule the streaming task instances
 * which are closer to the data nodes.
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    // Represents task schedule plan Id
    int taskSchedulePlanId = 0;
    Set<WorkerSchedulePlan> workerSchedulePlans = new HashSet<>();
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    Map<Integer, List<TaskInstanceId>> containerInstanceMap = dataLocalityStreamingSchedulingAlgorithm(graph, workerPlan.getNumberOfWorkers(), workerPlan);
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(containerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(containerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(containerInstanceMap, taskVertexSet);
    for (int containerId : containerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = containerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
        }
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
    }
    return new TaskSchedulePlan(taskSchedulePlanId, workerSchedulePlans);
}

Example 4 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class UserDefinedTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 *
 * @return TaskSchedulePlan
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    int taskSchedulePlanId = 0;
    // Allocate the task instances into the containers/workers
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    // To get the vertex set from the taskgraph
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    // Allocate the task instances into the logical containers.
    Map<Integer, List<TaskInstanceId>> userDefinedContainerInstanceMap = userDefinedSchedulingAlgorithm(graph, workerPlan.getNumberOfWorkers());
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    for (int containerId : userDefinedContainerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = userDefinedContainerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        // Create the container resource value based on the worker plan
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            LOG.fine("Worker (if loop):" + containerId + "\tRam:" + worker.getRam() + "\tDisk:" + worker.getDisk() + "\tCpu:" + worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            LOG.fine("Worker (else loop):" + containerId + "\tRam:" + containerRAMValue + "\tDisk:" + containerDiskValue + "\tCpu:" + containerCpuValue);
        }
        // Schedule the task instance plan into the task container plan.
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
    }
    return new TaskSchedulePlan(taskSchedulePlanId, workerSchedulePlans);
}

Example 5 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method addInstance.

public TaskSchedulePlanBuilder addInstance(Integer containerId, String taskName) throws TaskSchedulerException {
    initContainer(containerId);
    int taskId = taskIds.isEmpty() ? 1 : taskIds.last() + 1;
    int taskIndex = taskIndexes.get(taskName) != null ? taskIndexes.get(taskName).last() + 1 : 0;
    TaskInstanceId taskInstanceId = new TaskInstanceId(taskName, taskId, taskIndex);
    Resource resource = TaskScheduleUtils.getResourceRequirement(taskName, this.taskRamMap, this.instanceDefaultResourceValue, this.containerMaximumResourceValue, this.requestedContainerPadding);
    try {
        addToContainer(containers.get(containerId), new TaskInstancePlan(taskInstanceId, resource), taskIndexes, taskIds);
    } catch (TaskSchedulerException e) {
        throw new TaskSchedulerException(String.format("Insufficient container resources to add instance %s with resources %s to container %d.", taskInstanceId, resource, containerId), e);
    }
    LOG.info("Task id, index, name:" + taskId + "\t" + taskIndex + "\t" + taskName + "\tadded to Container:" + containers.get(containerId));
    return this;
}