Search in sources :

Example 11 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class ComputeGraphBuilder method build.

public ComputeGraph build() {
    ComputeGraph graph = new ComputeGraph();
    graph.setOperationMode(mode);
    graph.setGraphName(taskGraphName);
    graph.addGraphConstraints(graphConstraints);
    graph.addNodeConstraints(nodeConstraints);
    for (Map.Entry<String, Vertex> e : nodes.entrySet()) {
        graph.addTaskVertex(e.getKey(), e.getValue());
    }
    for (ComputeConnection c : computeConnections) {
        c.build(graph);
    }
    for (SourceConnection c : sourceConnections) {
        c.build(graph);
    }
    return graph;
}
Also used : Vertex(edu.iu.dsc.tws.api.compute.graph.Vertex) ComputeGraph(edu.iu.dsc.tws.api.compute.graph.ComputeGraph) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map)

Example 12 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method schedule.

/**
 * This is the base method which receives the dataflow taskgraph and the worker plan to allocate
 * the task instances to the appropriate workers with their required ram, disk, and cpu values.
 *
 * @param computeGraph
 * @param workerPlan   worker plan
 * @return
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph computeGraph, WorkerPlan workerPlan) {
    // Allocate the task instances into the containers/workers
    Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
    // To get the vertex set from the Collectible Name Settaskgraph
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(computeGraph.getTaskVertexSet());
    // Allocate the task instances into the logical containers.
    Map<Integer, List<TaskInstanceId>> batchContainerInstanceMap = batchSchedulingAlgorithm(computeGraph, workerPlan.getNumberOfWorkers());
    TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
    Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(batchContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(batchContainerInstanceMap, taskVertexSet);
    Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(batchContainerInstanceMap, taskVertexSet);
    for (int containerId : batchContainerInstanceMap.keySet()) {
        double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
        double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
        double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
        List<TaskInstanceId> taskTaskInstanceIds = batchContainerInstanceMap.get(containerId);
        Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
        for (TaskInstanceId id : taskTaskInstanceIds) {
            double instanceRAMValue = instancesRamMap.get(containerId).get(id);
            double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
            double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
            Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
            taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
            containerRAMValue += instanceRAMValue;
            containerDiskValue += instanceDiskValue;
            containerCpuValue += instanceDiskValue;
        }
        Worker worker = workerPlan.getWorker(containerId);
        Resource containerResource;
        // Create the container resource value based on the worker plan
        if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
            containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
        } else {
            containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
        }
        // Schedule the task instance plan into the task container plan.
        WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new LinkedHashSet<>(taskInstancePlanMap.values()), containerResource);
        workerSchedulePlans.add(taskWorkerSchedulePlan);
        if (dependentGraphs && index == 0) {
            workerIdList.add(containerId);
        }
    }
    index++;
    TaskSchedulePlan taskSchedulePlan = new TaskSchedulePlan(0, workerSchedulePlans);
    if (workerId == 0) {
        Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            Integer integer = entry.getKey();
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            LOG.fine("Graph Name:" + computeGraph.getGraphName() + "\tcontainer id:" + integer);
            for (TaskInstancePlan ip : containerPlanTaskInstances) {
                LOG.fine("Task Id:" + ip.getTaskId() + "\tIndex" + ip.getTaskIndex() + "\tName:" + ip.getTaskName());
            }
        }
    }
    return taskSchedulePlan;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) Vertex(edu.iu.dsc.tws.api.compute.graph.Vertex) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) TaskSchedulePlan(edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan) WorkerSchedulePlan(edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan) TaskInstanceMapCalculation(edu.iu.dsc.tws.tsched.spi.taskschedule.TaskInstanceMapCalculation) TaskInstancePlan(edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstancePlan) Worker(edu.iu.dsc.tws.api.compute.schedule.elements.Worker) ArrayList(java.util.ArrayList) List(java.util.List) Resource(edu.iu.dsc.tws.api.compute.schedule.elements.Resource) TaskInstanceId(edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map)

Example 13 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method batchSchedulingAlgorithm.

private Map<Integer, List<TaskInstanceId>> batchSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) throws TaskSchedulerException {
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    IntStream.range(0, numberOfContainers).forEach(i1 -> batchTaskAllocation.put(i1, new ArrayList<>()));
    int globalTaskIndex = 0;
    if (dependentGraphs) {
        for (Vertex vertex : taskVertexSet) {
            INode iNode = vertex.getTask();
            if (iNode instanceof Receptor) {
                validateReceptor(graph, vertex);
            }
            dependentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
            globalTaskIndex++;
        }
    } else {
        for (Vertex vertex : taskVertexSet) {
            INode iNode = vertex.getTask();
            if (iNode instanceof Collector) {
                ((Collector) iNode).getCollectibleNames().forEach(key -> collectibleNameMap.put(key, vertex.getParallelism()));
            } else if (iNode instanceof Receptor) {
                ((Receptor) iNode).getReceivableNames().forEach(key -> receivableNameMap.put(key, vertex.getParallelism()));
                validateParallelism();
            }
            independentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
            globalTaskIndex++;
        }
    }
    return batchTaskAllocation;
}
Also used : LinkedHashSet(java.util.LinkedHashSet) IntStream(java.util.stream.IntStream) TaskInstanceId(edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId) ComputeGraph(edu.iu.dsc.tws.api.compute.graph.ComputeGraph) TaskAttributes(edu.iu.dsc.tws.tsched.utils.TaskAttributes) INode(edu.iu.dsc.tws.api.compute.nodes.INode) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) TaskSchedulerContext(edu.iu.dsc.tws.tsched.spi.common.TaskSchedulerContext) Resource(edu.iu.dsc.tws.api.compute.schedule.elements.Resource) TreeSet(java.util.TreeSet) Twister2RuntimeException(edu.iu.dsc.tws.api.exceptions.Twister2RuntimeException) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) WorkerSchedulePlan(edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan) TaskInstanceMapCalculation(edu.iu.dsc.tws.tsched.spi.taskschedule.TaskInstanceMapCalculation) TaskSchedulePlan(edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan) Receptor(edu.iu.dsc.tws.api.compute.modifiers.Receptor) Map(java.util.Map) TaskInstancePlan(edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstancePlan) Collector(edu.iu.dsc.tws.api.compute.modifiers.Collector) LinkedHashSet(java.util.LinkedHashSet) Set(java.util.Set) Logger(java.util.logging.Logger) Vertex(edu.iu.dsc.tws.api.compute.graph.Vertex) List(java.util.List) ITaskScheduler(edu.iu.dsc.tws.api.compute.schedule.ITaskScheduler) Worker(edu.iu.dsc.tws.api.compute.schedule.elements.Worker) WorkerPlan(edu.iu.dsc.tws.api.compute.schedule.elements.WorkerPlan) Edge(edu.iu.dsc.tws.api.compute.graph.Edge) Comparator(java.util.Comparator) TaskSchedulerException(edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException) Vertex(edu.iu.dsc.tws.api.compute.graph.Vertex) INode(edu.iu.dsc.tws.api.compute.nodes.INode) Receptor(edu.iu.dsc.tws.api.compute.modifiers.Receptor) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) Collector(edu.iu.dsc.tws.api.compute.modifiers.Collector)

Example 14 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class ComputeConnection method build.

void build(ComputeGraph graph) {
    this.doAutoConnect();
    this.inputs.forEach((source, edges) -> {
        edges.forEach((edgeName, edge) -> {
            Vertex v1 = graph.vertex(nodeName);
            if (v1 == null) {
                throw new RuntimeException("Failed to connect non-existing task: " + nodeName);
            }
            Vertex v2 = graph.vertex(source);
            if (v2 == null) {
                throw new RuntimeException("Failed to connect non-existing task: " + source);
            }
            graph.addTaskEdge(v2, v1, edge);
        });
    });
}
Also used : Vertex(edu.iu.dsc.tws.api.compute.graph.Vertex)

Example 15 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class GraphBuilder method connect.

public GraphBuilder connect(String t1, String t2, Edge edge) {
    Vertex v1 = graph.vertex(t1);
    if (v1 == null) {
        throw new RuntimeException("Failed to connect non-existing task: " + t1);
    }
    Vertex v2 = graph.vertex(t2);
    if (v2 == null) {
        throw new RuntimeException("Failed to connect non-existing task: " + t2);
    }
    graph.addTaskEdge(v1, v2, edge);
    return this;
}
Also used : Vertex(edu.iu.dsc.tws.api.compute.graph.Vertex)

Aggregations

Vertex (edu.iu.dsc.tws.api.compute.graph.Vertex)27 LinkedHashMap (java.util.LinkedHashMap)15 ArrayList (java.util.ArrayList)12 HashMap (java.util.HashMap)11 List (java.util.List)11 TaskInstanceId (edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstanceId)10 Map (java.util.Map)10 Resource (edu.iu.dsc.tws.api.compute.schedule.elements.Resource)8 TaskInstancePlan (edu.iu.dsc.tws.api.compute.schedule.elements.TaskInstancePlan)8 TaskSchedulePlan (edu.iu.dsc.tws.api.compute.schedule.elements.TaskSchedulePlan)8 WorkerSchedulePlan (edu.iu.dsc.tws.api.compute.schedule.elements.WorkerSchedulePlan)8 Config (edu.iu.dsc.tws.api.config.Config)8 LinkedHashSet (java.util.LinkedHashSet)8 Worker (edu.iu.dsc.tws.api.compute.schedule.elements.Worker)7 TaskInstanceMapCalculation (edu.iu.dsc.tws.tsched.spi.taskschedule.TaskInstanceMapCalculation)7 TreeSet (java.util.TreeSet)6 TaskSchedulerException (edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException)4 TaskAttributes (edu.iu.dsc.tws.tsched.utils.TaskAttributes)4 HashSet (java.util.HashSet)4 Set (java.util.Set)4