Examples with Resource edu.iu.dsc.tws.api.compute.schedule.elements.Resource used on opensource projects

Example 6 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method initContainers.

/**
 * This method first initialize the container map values, task index values, and task id sets.
 */
private void initContainers() {
    assertResourceSettings();
    Map<Integer, Container> containerMap = this.containers;
    HashMap<String, TreeSet<Integer>> taskindexes = this.taskIndexes;
    TreeSet<Integer> taskids = this.taskIds;
    if (taskindexes == null) {
        taskindexes = new HashMap<>();
    }
    if (taskids == null) {
        taskids = new TreeSet<>();
    }
    if (containerMap == null) {
        if (this.previousTaskSchedulePlan == null) {
            containerMap = new HashMap<>();
            for (int containerId = 1; containerId <= numberOfContainers; containerId++) {
                containerMap.put(containerId, new Container(containerId, this.containerMaximumResourceValue, this.requestedContainerPadding));
            }
        } else {
            try {
                // containerMap = getContainers(this.previousTaskSchedulePlan);
                containerMap = getContainers(this.previousTaskSchedulePlan, this.requestedContainerPadding, taskindexes, taskids);
            } catch (TaskSchedulerException e) {
                throw new TaskSchedulerException("Could not initialize containers using existing packing plan", e);
            }
        }
    }
    if (this.numberOfContainers > containerMap.size()) {
        List<Scorer<Container>> scorers = new ArrayList<>();
        scorers.add(new ContainerIdScorer());
        List<Container> sortedContainers = sortContainers(scorers, containerMap.values());
        int nextContainerId = sortedContainers.get(sortedContainers.size() - 1).getContainerId() + 1;
        Resource capacity = containerMap.get(sortedContainers.get(0).getContainerId()).getResource();
        for (int i = 0; i < numberOfContainers - containerMap.size(); i++) {
            containerMap.put(nextContainerId, new Container(nextContainerId, capacity, this.requestedContainerPadding));
            nextContainerId++;
        }
    }
    this.taskIds = taskids;
    this.taskIndexes = taskindexes;
    this.containers = containerMap;
}

Example 7 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method getContainers.

/**
 * Get the containers based on the task schedule plan
 */
private Map<Integer, Container> getContainers(TaskSchedulePlan previoustaskschedulePlan) throws TaskSchedulerException {
    Map<Integer, Container> containerMap = new HashMap<>();
    Resource resource = previoustaskschedulePlan.getMaxContainerResources();
    for (WorkerSchedulePlan currentWorkerSchedulePlan : previoustaskschedulePlan.getContainers()) {
        Container container = new Container(currentWorkerSchedulePlan.getContainerId(), resource, requestedContainerPadding);
        for (TaskInstancePlan instancePlan : currentWorkerSchedulePlan.getTaskInstances()) {
            try {
                addToContainer(container, instancePlan, taskIndexes, taskIds);
            } catch (TaskSchedulerException e) {
                throw new TaskSchedulerException(String.format("Insufficient container resources to add instancePlan %s to container %s", instancePlan, container), e);
            }
        }
        containerMap.put(currentWorkerSchedulePlan.getContainerId(), container);
    }
    return containerMap;
}

Example 8 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method getContainers.

/**
 * Get the containers based on the task schedule plan
 * @param previoustaskschedulePlan
 * @return
 * @throws TaskSchedulerException
 */
private Map<Integer, Container> getContainers(TaskSchedulePlan previoustaskschedulePlan, int containerPadding, Map<String, TreeSet<Integer>> taskindexes, TreeSet<Integer> taskids) throws TaskSchedulerException {
    Map<Integer, Container> containerMap = new HashMap<>();
    Resource resource = previoustaskschedulePlan.getMaxContainerResources();
    for (WorkerSchedulePlan currentWorkerSchedulePlan : previoustaskschedulePlan.getContainers()) {
        Container container = new Container(currentWorkerSchedulePlan.getContainerId(), resource, containerPadding);
        for (TaskInstancePlan instancePlan : currentWorkerSchedulePlan.getTaskInstances()) {
            try {
                addToContainer(container, instancePlan, taskindexes, taskids);
            } catch (TaskSchedulerException e) {
                throw new TaskSchedulerException(String.format("Insufficient container resources to add instancePlan %s to container %s", instancePlan, container), e);
            }
        }
        containerMap.put(currentWorkerSchedulePlan.getContainerId(), container);
    }
    return containerMap;
}

Example 9 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method buildContainerPlans.

private Set<WorkerSchedulePlan> buildContainerPlans(Map<Integer, Container> containerValue, Map<String, Double> taskramMap, Resource instdefaultresourcevalue) {
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    try {
        for (Integer containerId : containerValue.keySet()) {
            Container container = containerValue.get(containerId);
            if (container.getTaskInstances().size() == 0) {
                continue;
            }
            double containerRAMValue = 0.0;
            double containerDiskValue = 0.0;
            double containerCPUValue = 0.0;
            Set<TaskInstancePlan> taskInstancePlans = new HashSet<>();
            for (TaskInstancePlan taskInstancePlan : container.getTaskInstances()) {
                TaskInstanceId taskInstanceId = new TaskInstanceId(taskInstancePlan.getTaskName(), taskInstancePlan.getTaskId(), taskInstancePlan.getTaskIndex());
                double instanceRAMValue;
                if (taskramMap.containsKey(taskInstanceId.getTaskName())) {
                    instanceRAMValue = taskramMap.get(taskInstanceId.getTaskName());
                } else {
                    instanceRAMValue = instdefaultresourcevalue.getRam();
                }
                containerRAMValue += instanceRAMValue;
                double instanceDiskValue = instdefaultresourcevalue.getDisk();
                containerDiskValue += instanceDiskValue;
                double instanceCPUValue = instdefaultresourcevalue.getCpu();
                containerCPUValue += instanceCPUValue;
                LOG.fine("Resource Container Values:" + "Ram Value:" + containerRAMValue + "\t" + "Cpu Value:" + containerCPUValue + "\t" + "Disk Value:" + containerDiskValue);
                Resource resource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
                taskInstancePlans.add(new TaskInstancePlan(taskInstanceId.getTaskName(), taskInstanceId.getTaskId(), taskInstanceId.getTaskIndex(), resource));
            }
            containerCPUValue += (requestedContainerPadding * containerCPUValue) / 100;
            containerRAMValue += containerRAMValue + requestedContainerPadding;
            containerDiskValue += containerDiskValue + requestedContainerPadding;
            Resource resource = new Resource(containerRAMValue, containerDiskValue, containerCPUValue);
            WorkerSchedulePlan workerSchedulePlan = new WorkerSchedulePlan(containerId, taskInstancePlans, resource);
            workerSchedulePlans.add(workerSchedulePlan);
        }
    } catch (TaskSchedulerException ne) {
        throw new RuntimeException("Exception Occured" + ne.getMessage());
    }
    return workerSchedulePlans;
}

Example 10 with Resource

use of edu.iu.dsc.tws.api.compute.schedule.elements.Resource in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 *
 * @param computeGraph
 * @param workerPlan   worker plan
 * @return
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph computeGraph, WorkerPlan workerPlan) {
    // Allocate the task instances into the containers/workers
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    // To get the vertex set from the Collectible Name Settaskgraph
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(computeGraph.getTaskVertexSet());
    // Allocate the task instances into the logical containers.
    Map<Integer, List<TaskInstanceId>> batchContainerInstanceMap = batchSchedulingAlgorithm(computeGraph, workerPlan.getNumberOfWorkers());
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(batchContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(batchContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(batchContainerInstanceMap, taskVertexSet);
    for (int containerId : batchContainerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = batchContainerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        // Create the container resource value based on the worker plan
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
        }
        // Schedule the task instance plan into the task container plan.
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new LinkedHashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
        if (dependentGraphs && index == 0) {
            workerIdList.add(containerId);
        }
    }
    index++;
    TaskSchedulePlan taskSchedulePlan = new TaskSchedulePlan(0, workerSchedulePlans);
    if (workerId == 0) {
        Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            Integer integer = entry.getKey();
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            LOG.fine("Graph Name:" + computeGraph.getGraphName() + "\tcontainer id:" + integer);
            for (TaskInstancePlan ip : containerPlanTaskInstances) {
                LOG.fine("Task Id:" + ip.getTaskId() + "\tIndex" + ip.getTaskIndex() + "\tName:" + ip.getTaskName());
            }
        }
    }
    return taskSchedulePlan;
}