use of edu.iu.dsc.tws.api.compute.nodes.ICompute in project twister2 by DSC-SPIDAL.
the class ExecutionPlanBuilder method createInstances.
/**
* Create an instance of a task,
*
* @param cfg the configuration
* @param ip instance plan
* @param vertex vertex
*/
private INodeInstance createInstances(Config cfg, String taskGraphName, TaskInstancePlan ip, Vertex vertex, OperationMode operationMode, Map<String, Set<String>> inEdges, Map<String, String> outEdges, TaskSchedulePlan taskSchedule, long tasksVersion) {
// lets add the task
byte[] taskBytes = Utils.serialize(vertex.getTask());
INode newInstance = (INode) Utils.deserialize(taskBytes);
int taskId = taskIdGenerator.generateGlobalTaskId(ip.getTaskId(), ip.getTaskIndex());
if (operationMode.equals(OperationMode.BATCH)) {
if (newInstance instanceof ICompute) {
TaskBatchInstance v = new TaskBatchInstance((ICompute) newInstance, new ArrayBlockingQueue<>(ExecutorContext.instanceQueueLowWaterMark(cfg)), new LinkedBlockingQueue<>(), cfg, vertex.getName(), ip.getTaskId(), taskId, ip.getTaskIndex(), vertex.getParallelism(), workerId, vertex.getConfig().toMap(), inEdges, outEdges, taskSchedule, this.checkpointingClient, taskGraphName, tasksVersion);
batchTaskInstances.put(vertex.getName(), taskId, v);
return v;
} else if (newInstance instanceof ISource) {
SourceBatchInstance v = new SourceBatchInstance((ISource) newInstance, new LinkedBlockingQueue<>(), cfg, vertex.getName(), ip.getTaskId(), taskId, ip.getTaskIndex(), vertex.getParallelism(), workerId, vertex.getConfig().toMap(), outEdges, taskSchedule, this.checkpointingClient, taskGraphName, tasksVersion);
batchSourceInstances.put(vertex.getName(), taskId, v);
return v;
} else {
throw new RuntimeException("Un-known type");
}
} else if (operationMode.equals(OperationMode.STREAMING)) {
if (newInstance instanceof ICompute) {
TaskStreamingInstance v = new TaskStreamingInstance((ICompute) newInstance, new ArrayBlockingQueue<>(ExecutorContext.instanceQueueLowWaterMark(cfg)), new LinkedBlockingQueue<>(), cfg, vertex.getName(), ip.getTaskId(), taskId, ip.getTaskIndex(), vertex.getParallelism(), workerId, vertex.getConfig().toMap(), inEdges, outEdges, taskSchedule, this.checkpointingClient, taskGraphName, tasksVersion);
streamingTaskInstances.put(vertex.getName(), taskId, v);
return v;
} else if (newInstance instanceof ISource) {
SourceStreamingInstance v = new SourceStreamingInstance((ISource) newInstance, new LinkedBlockingQueue<>(), cfg, vertex.getName(), ip.getTaskId(), taskId, ip.getTaskIndex(), vertex.getParallelism(), workerId, vertex.getConfig().toMap(), outEdges, taskSchedule, this.checkpointingClient, taskGraphName, tasksVersion);
streamingSourceInstances.put(vertex.getName(), taskId, v);
return v;
} else {
throw new RuntimeException("Un-known type");
}
} else {
// GRAPH Structures must be implemented here
return null;
}
}
use of edu.iu.dsc.tws.api.compute.nodes.ICompute in project twister2 by DSC-SPIDAL.
the class ExecutionPlanBuilder method build.
@Override
public ExecutionPlan build(Config cfg, ComputeGraph taskGraph, TaskSchedulePlan taskSchedule) {
// we need to build the task plan
LogicalPlan logicalPlan = TaskPlanBuilder.build(workerId, workerInfoList, taskSchedule, taskIdGenerator);
ParallelOperationFactory opFactory = new ParallelOperationFactory(cfg, network, logicalPlan);
Map<Integer, WorkerSchedulePlan> containersMap = taskSchedule.getContainersMap();
WorkerSchedulePlan conPlan = containersMap.get(workerId);
if (conPlan == null) {
LOG.log(Level.INFO, "Cannot find worker in the task plan: " + workerId);
return null;
}
ExecutionPlan execution = new ExecutionPlan();
Set<TaskInstancePlan> instancePlan = conPlan.getTaskInstances();
long tasksVersion = 0L;
if (CheckpointingContext.isCheckpointingEnabled(cfg)) {
Set<Integer> globalTasks = Collections.emptySet();
if (workerId == 0) {
globalTasks = containersMap.values().stream().flatMap(containerPlan -> containerPlan.getTaskInstances().stream()).filter(ip -> taskGraph.vertex(ip.getTaskName()).getTask() instanceof CheckpointableTask && !(taskGraph.vertex(ip.getTaskName()).getTask() instanceof CheckpointingSGatherSink)).map(TaskInstancePlan::getTaskId).collect(Collectors.toSet());
}
try {
Checkpoint.FamilyInitializeResponse familyInitializeResponse = this.checkpointingClient.initFamily(workerId, containersMap.size(), taskGraph.getGraphName(), globalTasks);
tasksVersion = familyInitializeResponse.getVersion();
} catch (BlockingSendException e) {
throw new RuntimeException("Failed to register tasks with Checkpoint Manager", e);
}
LOG.info("Tasks will start with version " + tasksVersion);
}
// for each task we are going to create the communications
for (TaskInstancePlan ip : instancePlan) {
Vertex v = taskGraph.vertex(ip.getTaskName());
Map<String, Set<String>> inEdges = new HashMap<>();
Map<String, String> outEdges = new HashMap<>();
if (v == null) {
throw new RuntimeException("Non-existing task scheduled: " + ip.getTaskName());
}
INode node = v.getTask();
if (node instanceof ICompute || node instanceof ISource) {
// lets get the communication
Set<Edge> edges = taskGraph.outEdges(v);
// now lets create the communication object
for (Edge e : edges) {
Vertex child = taskGraph.childOfTask(v, e.getName());
// lets figure out the parents task id
Set<Integer> srcTasks = taskIdGenerator.getTaskIds(v, ip.getTaskId());
Set<Integer> tarTasks = taskIdGenerator.getTaskIds(child, getTaskIdOfTask(child.getName(), taskSchedule));
Map<Integer, Integer> srcGlobalToIndex = taskIdGenerator.getGlobalTaskToIndex(v, ip.getTaskId());
Map<Integer, Integer> tarGlobaToIndex = taskIdGenerator.getGlobalTaskToIndex(child, getTaskIdOfTask(child.getName(), taskSchedule));
createCommunication(child, e, v, srcTasks, tarTasks, srcGlobalToIndex, tarGlobaToIndex);
outEdges.put(e.getName(), child.getName());
}
}
if (node instanceof ICompute) {
// lets get the parent tasks
Set<Edge> parentEdges = taskGraph.inEdges(v);
for (Edge e : parentEdges) {
Vertex parent = taskGraph.getParentOfTask(v, e.getName());
// lets figure out the parents task id
Set<Integer> srcTasks = taskIdGenerator.getTaskIds(parent, getTaskIdOfTask(parent.getName(), taskSchedule));
Set<Integer> tarTasks = taskIdGenerator.getTaskIds(v, ip.getTaskId());
Map<Integer, Integer> srcGlobalToIndex = taskIdGenerator.getGlobalTaskToIndex(parent, getTaskIdOfTask(parent.getName(), taskSchedule));
Map<Integer, Integer> tarGlobalToIndex = taskIdGenerator.getGlobalTaskToIndex(v, ip.getTaskId());
createCommunication(v, e, parent, srcTasks, tarTasks, srcGlobalToIndex, tarGlobalToIndex);
// if we are a grouped edge, we have to use the group name
String inEdge;
if (e.getTargetEdge() == null) {
inEdge = e.getName();
} else {
inEdge = e.getTargetEdge();
}
Set<String> parents = inEdges.get(inEdge);
if (parents == null) {
parents = new HashSet<>();
}
parents.add(inEdge);
inEdges.put(inEdge, parents);
}
}
// lets create the instance
INodeInstance iNodeInstance = createInstances(cfg, taskGraph.getGraphName(), ip, v, taskGraph.getOperationMode(), inEdges, outEdges, taskSchedule, tasksVersion);
// add to execution
execution.addNodes(v.getName(), taskIdGenerator.generateGlobalTaskId(ip.getTaskId(), ip.getTaskIndex()), iNodeInstance);
}
// now lets create the queues and start the execution
for (Table.Cell<String, String, Communication> cell : parOpTable.cellSet()) {
Communication c = cell.getValue();
// lets create the communication
OperationMode operationMode = taskGraph.getOperationMode();
IParallelOperation op;
assert c != null;
c.build();
if (c.getEdge().size() == 1) {
op = opFactory.build(c.getEdge(0), c.getSourceTasks(), c.getTargetTasks(), operationMode, c.srcGlobalToIndex, c.tarGlobalToIndex);
} else if (c.getEdge().size() > 1) {
// just join op for now. Could change in the future
// here the sources should be separated out for left and right edge
Set<Integer> sourceTasks = c.getSourceTasks();
Set<Integer> leftSources = new HashSet<>();
Set<Integer> rightSources = new HashSet<>();
if (!sourceTasks.isEmpty()) {
// just to safely do .get() calls without isPresent()
int minBin = (sourceTasks.stream().min(Integer::compareTo).get() / TaskIdGenerator.TASK_OFFSET) * TaskIdGenerator.TASK_OFFSET;
for (Integer source : sourceTasks) {
if ((source / TaskIdGenerator.TASK_OFFSET) * TaskIdGenerator.TASK_OFFSET == minBin) {
leftSources.add(source);
} else {
rightSources.add(source);
}
}
}
// now determine, which task is connected to which edge
Edge leftEdge = c.getEdge(0);
Edge rightEdge = c.getEdge(1);
op = opFactory.build(leftEdge, rightEdge, leftSources, rightSources, c.getTargetTasks(), operationMode, c.srcGlobalToIndex, c.tarGlobalToIndex);
} else {
throw new RuntimeException("Cannot have communication with 0 edges");
}
// now lets check the sources and targets that are in this executor
Set<Integer> sourcesOfThisWorker = intersectionOfTasks(conPlan, c.getSourceTasks());
Set<Integer> targetsOfThisWorker = intersectionOfTasks(conPlan, c.getTargetTasks());
// we use the target edge as the group name
String targetEdge;
if (c.getEdge().size() > 1) {
targetEdge = c.getEdge(0).getTargetEdge();
} else {
targetEdge = c.getEdge(0).getName();
}
// so along with the operation mode, the windowing mode must be tested
if (operationMode == OperationMode.STREAMING) {
for (Integer i : sourcesOfThisWorker) {
boolean found = false;
// we can have multiple source tasks for an operation
for (int sIndex = 0; sIndex < c.getSourceTask().size(); sIndex++) {
String sourceTask = c.getSourceTask().get(sIndex);
if (streamingTaskInstances.contains(sourceTask, i)) {
TaskStreamingInstance taskStreamingInstance = streamingTaskInstances.get(sourceTask, i);
taskStreamingInstance.registerOutParallelOperation(c.getEdge(sIndex).getName(), op);
op.registerSync(i, taskStreamingInstance);
found = true;
} else if (streamingSourceInstances.contains(sourceTask, i)) {
SourceStreamingInstance sourceStreamingInstance = streamingSourceInstances.get(sourceTask, i);
sourceStreamingInstance.registerOutParallelOperation(c.getEdge(sIndex).getName(), op);
found = true;
}
if (!found) {
throw new RuntimeException("Not found: " + c.getSourceTask());
}
}
}
// we only have one target task always
for (Integer i : targetsOfThisWorker) {
if (streamingTaskInstances.contains(c.getTargetTask(), i)) {
TaskStreamingInstance taskStreamingInstance = streamingTaskInstances.get(c.getTargetTask(), i);
op.register(i, taskStreamingInstance.getInQueue());
taskStreamingInstance.registerInParallelOperation(targetEdge, op);
op.registerSync(i, taskStreamingInstance);
} else {
throw new RuntimeException("Not found: " + c.getTargetTask());
}
}
execution.addOps(op);
}
if (operationMode == OperationMode.BATCH) {
for (Integer i : sourcesOfThisWorker) {
boolean found = false;
// we can have multiple source tasks for an operation
for (int sIndex = 0; sIndex < c.getSourceTask().size(); sIndex++) {
String sourceTask = c.getSourceTask().get(sIndex);
if (batchTaskInstances.contains(sourceTask, i)) {
TaskBatchInstance taskBatchInstance = batchTaskInstances.get(sourceTask, i);
taskBatchInstance.registerOutParallelOperation(c.getEdge(sIndex).getName(), op);
found = true;
} else if (batchSourceInstances.contains(sourceTask, i)) {
SourceBatchInstance sourceBatchInstance = batchSourceInstances.get(sourceTask, i);
sourceBatchInstance.registerOutParallelOperation(c.getEdge(sIndex).getName(), op);
found = true;
}
}
if (!found) {
throw new RuntimeException("Not found: " + c.getSourceTask());
}
}
for (Integer i : targetsOfThisWorker) {
if (batchTaskInstances.contains(c.getTargetTask(), i)) {
TaskBatchInstance taskBatchInstance = batchTaskInstances.get(c.getTargetTask(), i);
op.register(i, taskBatchInstance.getInQueue());
taskBatchInstance.registerInParallelOperation(targetEdge, op);
op.registerSync(i, taskBatchInstance);
} else {
throw new RuntimeException("Not found: " + c.getTargetTask());
}
}
execution.addOps(op);
}
}
return execution;
}
use of edu.iu.dsc.tws.api.compute.nodes.ICompute in project twister2 by DSC-SPIDAL.
the class BTAllReduceExample method buildTaskGraph.
@Override
public ComputeGraphBuilder buildTaskGraph() {
List<Integer> taskStages = jobParameters.getTaskStages();
int sourceParallelism = taskStages.get(0);
int sinkParallelism = taskStages.get(1);
String edge = "edge";
BaseSource g = new SourceTask(edge);
ICompute r = new AllReduceSinkTask();
computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
computeConnection = computeGraphBuilder.addCompute(SINK, r, sinkParallelism);
computeConnection.allreduce(SOURCE).viaEdge(edge).withOperation(Op.SUM, MessageTypes.INTEGER_ARRAY);
return computeGraphBuilder;
}
use of edu.iu.dsc.tws.api.compute.nodes.ICompute in project twister2 by DSC-SPIDAL.
the class BTReduceExample method buildTaskGraph.
@Override
public ComputeGraphBuilder buildTaskGraph() {
List<Integer> taskStages = jobParameters.getTaskStages();
int sourceParallelism = taskStages.get(0);
int sinkParallelism = taskStages.get(1);
String edge = "edge";
BaseSource g = new SourceTask(edge);
ICompute r = new ReduceSinkTask();
computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
computeConnection = computeGraphBuilder.addCompute(SINK, r, sinkParallelism);
computeConnection.reduce(SOURCE).viaEdge(edge).withOperation(Op.SUM, MessageTypes.INTEGER_ARRAY);
return computeGraphBuilder;
}
use of edu.iu.dsc.tws.api.compute.nodes.ICompute in project twister2 by DSC-SPIDAL.
the class STAllGatherExample method buildTaskGraph.
@Override
public ComputeGraphBuilder buildTaskGraph() {
List<Integer> taskStages = jobParameters.getTaskStages();
int psource = taskStages.get(0);
int psink = taskStages.get(1);
MessageType dataType = MessageTypes.INTEGER_ARRAY;
String edge = "edge";
BaseSource g = new SourceTask(edge);
ICompute r = new AllGatherSinkTask();
computeGraphBuilder.addSource(SOURCE, g, psource);
computeConnection = computeGraphBuilder.addCompute(SINK, r, psink);
computeConnection.allgather(SOURCE).viaEdge(edge).withDataType(dataType);
return computeGraphBuilder;
}
Aggregations