Examples with Vertex edu.iu.dsc.tws.api.compute.graph.Vertex used on opensource projects

Example 6 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class UserDefinedTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 *
 * @return TaskSchedulePlan
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    int taskSchedulePlanId = 0;
    // Allocate the task instances into the containers/workers
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    // To get the vertex set from the taskgraph
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    // Allocate the task instances into the logical containers.
    Map<Integer, List<TaskInstanceId>> userDefinedContainerInstanceMap = userDefinedSchedulingAlgorithm(graph, workerPlan.getNumberOfWorkers());
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(userDefinedContainerInstanceMap, taskVertexSet);
    for (int containerId : userDefinedContainerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = userDefinedContainerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        // Create the container resource value based on the worker plan
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            LOG.fine("Worker (if loop):" + containerId + "\tRam:" + worker.getRam() + "\tDisk:" + worker.getDisk() + "\tCpu:" + worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            LOG.fine("Worker (else loop):" + containerId + "\tRam:" + containerRAMValue + "\tDisk:" + containerDiskValue + "\tCpu:" + containerCpuValue);
        }
        // Schedule the task instance plan into the task container plan.
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
    }
    return new TaskSchedulePlan(taskSchedulePlanId, workerSchedulePlans);
}

Example 7 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getTaskNetworkMap.

/**
 * This method retrieve the set of task vertices and check if the task vertex has the user
 * specified network value. If the user doesn't specify the required network configuration it will
 * assign the default network value from the task configuration file and store it in the map.
 */
public Map<String, Double> getTaskNetworkMap(Set<Vertex> taskVertices) {
    Map<String, Double> taskNetworkMap = new LinkedHashMap<>();
    Object network;
    double requiredNetwork;
    for (Vertex task : taskVertices) {
        Config config = task.getConfig();
        if (config.get("Network") != null) {
            network = config.get("Network");
            requiredNetwork = (double) ((Integer) network);
        } else {
            requiredNetwork = TaskSchedulerContext.taskInstanceNetwork(config);
        }
        taskNetworkMap.put(task.getName(), requiredNetwork);
    }
    return taskNetworkMap;
}

Example 8 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getTaskCPUMap.

/**
 * This method retrieve the set of task vertices and check if the task vertex has the user
 * specified cpu value. If the user doesn't specify the required cpu configuration it will assign
 * the default cpu value from the task configuration file and store it in the map.
 */
public Map<String, Double> getTaskCPUMap(Set<Vertex> taskVertices) {
    Map<String, Double> taskCPUMap = new LinkedHashMap<>();
    Object cpu;
    double requiredCpu;
    for (Vertex task : taskVertices) {
        Config config = task.getConfig();
        if (config.get("Cpu") != null) {
            cpu = config.get("Cpu");
            requiredCpu = (double) ((Integer) cpu);
        } else {
            requiredCpu = TaskSchedulerContext.taskInstanceCpu(config);
        }
        taskCPUMap.put(task.getName(), requiredCpu);
    }
    return taskCPUMap;
}

Example 9 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class TaskAttributes method getParallelTaskMap.

/**
 * This method is mainly to generate the parallel task map (maintain order) for the task vertex.
 * If the user specifies the parallelism value greater than or equal "1" will be considered as a
 * parallelism value. Otherwise, the system assign the default parallelism value to the task
 * vertex from the task scheduling configuration file.
 */
public Map<String, Integer> getParallelTaskMap(Set<Vertex> iTaskSet, Map<String, Map<String, String>> nodeConstraintsMap) {
    Map<String, Integer> parallelTaskMap = new LinkedHashMap<>();
    for (Vertex taskVertex : iTaskSet) {
        Config config = taskVertex.getConfig();
        String taskName = taskVertex.getName();
        int parallelTaskCount;
        if (!nodeConstraintsMap.get(taskName).isEmpty()) {
            Map<String, String> vertexMap = nodeConstraintsMap.get(taskName);
            if (vertexMap.containsKey(Context.TWISTER2_TASK_INSTANCE_ODD_PARALLELISM)) {
                parallelTaskCount = Integer.valueOf(String.valueOf(vertexMap.get(Context.TWISTER2_TASK_INSTANCE_ODD_PARALLELISM)));
            } else {
                parallelTaskCount = taskVertex.getParallelism();
            }
        } else {
            if (taskVertex.getParallelism() >= 1) {
                parallelTaskCount = taskVertex.getParallelism();
            } else {
                parallelTaskCount = TaskSchedulerContext.taskParallelism(config);
            }
        }
        parallelTaskMap.put(taskName, parallelTaskCount);
    }
    return parallelTaskMap;
}

Example 10 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class GraphBuilder method setParallelism.

public GraphBuilder setParallelism(String taskName, int parallel) {
    Vertex v = graph.vertex(taskName);
    if (v == null) {
        throw new RuntimeException("Failed to add configuration to non-existing task: " + taskName);
    }
    v.setParallelism(parallel);
    return this;
}