use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.
the class ComputeGraphBuilder method build.
public ComputeGraph build() {
ComputeGraph graph = new ComputeGraph();
graph.setOperationMode(mode);
graph.setGraphName(taskGraphName);
graph.addGraphConstraints(graphConstraints);
graph.addNodeConstraints(nodeConstraints);
for (Map.Entry<String, Vertex> e : nodes.entrySet()) {
graph.addTaskVertex(e.getKey(), e.getValue());
}
for (ComputeConnection c : computeConnections) {
c.build(graph);
}
for (SourceConnection c : sourceConnections) {
c.build(graph);
}
return graph;
}
use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.
the class BatchTaskScheduler method schedule.
/**
* This is the base method which receives the dataflow taskgraph and the worker plan to allocate
* the task instances to the appropriate workers with their required ram, disk, and cpu values.
*
* @param computeGraph
* @param workerPlan worker plan
* @return
*/
@Override
public TaskSchedulePlan schedule(ComputeGraph computeGraph, WorkerPlan workerPlan) {
// Allocate the task instances into the containers/workers
Set<WorkerSchedulePlan> workerSchedulePlans = new LinkedHashSet<>();
// To get the vertex set from the Collectible Name Settaskgraph
Set<Vertex> taskVertexSet = new LinkedHashSet<>(computeGraph.getTaskVertexSet());
// Allocate the task instances into the logical containers.
Map<Integer, List<TaskInstanceId>> batchContainerInstanceMap = batchSchedulingAlgorithm(computeGraph, workerPlan.getNumberOfWorkers());
TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(batchContainerInstanceMap, taskVertexSet);
Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(batchContainerInstanceMap, taskVertexSet);
Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(batchContainerInstanceMap, taskVertexSet);
for (int containerId : batchContainerInstanceMap.keySet()) {
double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
List<TaskInstanceId> taskTaskInstanceIds = batchContainerInstanceMap.get(containerId);
Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
for (TaskInstanceId id : taskTaskInstanceIds) {
double instanceRAMValue = instancesRamMap.get(containerId).get(id);
double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
containerRAMValue += instanceRAMValue;
containerDiskValue += instanceDiskValue;
containerCpuValue += instanceDiskValue;
}
Worker worker = workerPlan.getWorker(containerId);
Resource containerResource;
// Create the container resource value based on the worker plan
if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
} else {
containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
}
// Schedule the task instance plan into the task container plan.
WorkerSchedulePlan taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new LinkedHashSet<>(taskInstancePlanMap.values()), containerResource);
workerSchedulePlans.add(taskWorkerSchedulePlan);
if (dependentGraphs && index == 0) {
workerIdList.add(containerId);
}
}
index++;
TaskSchedulePlan taskSchedulePlan = new TaskSchedulePlan(0, workerSchedulePlans);
if (workerId == 0) {
Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
Integer integer = entry.getKey();
WorkerSchedulePlan workerSchedulePlan = entry.getValue();
Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
LOG.fine("Graph Name:" + computeGraph.getGraphName() + "\tcontainer id:" + integer);
for (TaskInstancePlan ip : containerPlanTaskInstances) {
LOG.fine("Task Id:" + ip.getTaskId() + "\tIndex" + ip.getTaskIndex() + "\tName:" + ip.getTaskName());
}
}
}
return taskSchedulePlan;
}
use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.
the class BatchTaskScheduler method batchSchedulingAlgorithm.
private Map<Integer, List<TaskInstanceId>> batchSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) throws TaskSchedulerException {
Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
orderedTaskSet.addAll(taskVertexSet);
IntStream.range(0, numberOfContainers).forEach(i1 -> batchTaskAllocation.put(i1, new ArrayList<>()));
int globalTaskIndex = 0;
if (dependentGraphs) {
for (Vertex vertex : taskVertexSet) {
INode iNode = vertex.getTask();
if (iNode instanceof Receptor) {
validateReceptor(graph, vertex);
}
dependentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
globalTaskIndex++;
}
} else {
for (Vertex vertex : taskVertexSet) {
INode iNode = vertex.getTask();
if (iNode instanceof Collector) {
((Collector) iNode).getCollectibleNames().forEach(key -> collectibleNameMap.put(key, vertex.getParallelism()));
} else if (iNode instanceof Receptor) {
((Receptor) iNode).getReceivableNames().forEach(key -> receivableNameMap.put(key, vertex.getParallelism()));
validateParallelism();
}
independentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
globalTaskIndex++;
}
}
return batchTaskAllocation;
}
use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.
the class ComputeConnection method build.
void build(ComputeGraph graph) {
this.doAutoConnect();
this.inputs.forEach((source, edges) -> {
edges.forEach((edgeName, edge) -> {
Vertex v1 = graph.vertex(nodeName);
if (v1 == null) {
throw new RuntimeException("Failed to connect non-existing task: " + nodeName);
}
Vertex v2 = graph.vertex(source);
if (v2 == null) {
throw new RuntimeException("Failed to connect non-existing task: " + source);
}
graph.addTaskEdge(v2, v1, edge);
});
});
}
use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.
the class GraphBuilder method connect.
public GraphBuilder connect(String t1, String t2, Edge edge) {
Vertex v1 = graph.vertex(t1);
if (v1 == null) {
throw new RuntimeException("Failed to connect non-existing task: " + t1);
}
Vertex v2 = graph.vertex(t2);
if (v2 == null) {
throw new RuntimeException("Failed to connect non-existing task: " + t2);
}
graph.addTaskEdge(v1, v2, edge);
return this;
}
Aggregations