use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.
the class KMeansConnectedDataflowExample method generateThirdJob.
private static DataFlowGraph generateThirdJob(Config config, int parallelismValue, int instances, int iterations, int dimension, DataFlowJobConfig jobConfig) {
KMeansSourceTask kMeansSourceTask = new KMeansSourceTask(dimension);
KMeansAllReduceTask kMeansAllReduceTask = new KMeansAllReduceTask();
ComputeGraphBuilder kmeansComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
// Add source, and sink tasks to the task graph builder for the third task graph
kmeansComputeGraphBuilder.addSource("kmeanssource", kMeansSourceTask, parallelismValue);
ComputeConnection kMeanscomputeConnection = kmeansComputeGraphBuilder.addCompute("kmeanssink", kMeansAllReduceTask, parallelismValue);
// Creating the communication edges between the tasks for the third task graph
kMeanscomputeConnection.allreduce("kmeanssource").viaEdge("all-reduce").withReductionFunction(new CentroidAggregator()).withDataType(MessageTypes.OBJECT);
kmeansComputeGraphBuilder.setMode(OperationMode.BATCH);
kmeansComputeGraphBuilder.setTaskGraphName("kmeansTG");
ComputeGraph thirdGraph = kmeansComputeGraphBuilder.build();
DataFlowGraph job = DataFlowGraph.newSubGraphJob("kmeansTG", thirdGraph).setWorkers(instances).addDataFlowJobConfig(jobConfig).setGraphType("iterative").setIterations(iterations);
return job;
}
use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.
the class KMeansConnectedDataflowExample method generateFirstJob.
private static DataFlowGraph generateFirstJob(Config config, int parallelismValue, String dataDirectory, int dimension, int dsize, int instances, DataFlowJobConfig jobConfig) {
DataObjectSource dataObjectSource = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory);
KMeansDataObjectCompute dataObjectCompute = new KMeansDataObjectCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue, dimension);
KMeansDataObjectDirectSink dataObjectSink = new KMeansDataObjectDirectSink("points");
ComputeGraphBuilder datapointsComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
// Add source, compute, and sink tasks to the task graph builder for the first task graph
datapointsComputeGraphBuilder.addSource("datapointsource", dataObjectSource, parallelismValue);
ComputeConnection datapointComputeConnection = datapointsComputeGraphBuilder.addCompute("datapointcompute", dataObjectCompute, parallelismValue);
ComputeConnection firstGraphComputeConnection = datapointsComputeGraphBuilder.addCompute("datapointsink", dataObjectSink, parallelismValue);
// Creating the communication edges between the tasks for the second task graph
datapointComputeConnection.direct("datapointsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
firstGraphComputeConnection.direct("datapointcompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
datapointsComputeGraphBuilder.setMode(OperationMode.BATCH);
datapointsComputeGraphBuilder.setTaskGraphName("datapointsTG");
ComputeGraph firstGraph = datapointsComputeGraphBuilder.build();
DataFlowGraph job = DataFlowGraph.newSubGraphJob("datapointsTG", firstGraph).setWorkers(instances).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
return job;
}
use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.
the class TaskGraphBuildTest method createGraphWithEdgeName.
private ComputeGraph createGraphWithEdgeName(String edgeName) {
TestSource testSource = new TestSource();
TestSink1 testCompute = new TestSink1();
TestSink2 testSink = new TestSink2();
ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(getConfig());
computeGraphBuilder.addSource("source", testSource, 4);
ComputeConnection computeConnection = computeGraphBuilder.addCompute("compute", testCompute, 4);
computeConnection.partition("source").viaEdge(edgeName).withDataType(MessageTypes.OBJECT);
ComputeConnection rc = computeGraphBuilder.addCompute("sink", testSink, 1);
rc.allreduce("compute").viaEdge(edgeName).withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
ComputeGraph graph = computeGraphBuilder.build();
return graph;
}
use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.
the class SvmSgdIterativeRunner method buildSvmSgdTestingTG.
private ComputeGraph buildSvmSgdTestingTG() {
iterativePredictionDataStreamer = new IterativePredictionDataStreamer(this.svmJobParameters.getFeatures(), this.operationMode, this.svmJobParameters.isDummy(), this.binaryBatchModel);
iterativeSVMAccuracyReduce = new IterativeSVMAccuracyReduce(this.operationMode);
testingBuilder.addSource(Constants.SimpleGraphConfig.PREDICTION_SOURCE_TASK, iterativePredictionDataStreamer, dataStreamerParallelism);
ComputeConnection svmComputeConnection = testingBuilder.addCompute(Constants.SimpleGraphConfig.PREDICTION_REDUCE_TASK, iterativeSVMAccuracyReduce, dataStreamerParallelism);
svmComputeConnection.allreduce(Constants.SimpleGraphConfig.PREDICTION_SOURCE_TASK).viaEdge(Constants.SimpleGraphConfig.PREDICTION_EDGE).withReductionFunction(new IterativeAccuracyReduceFunction()).withDataType(MessageTypes.DOUBLE);
testingBuilder.setMode(operationMode);
testingBuilder.setTaskGraphName(IterativeSVMConstants.ITERATIVE_PREDICTION_TASK_GRAPH);
return testingBuilder.build();
}
use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.
the class SourceTaskDataLoader method execute.
@Override
public void execute() {
getParams();
/*
* First data is loaded from files
* */
ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
// DataObjectSource sourceTask = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE,
// dataSource);
// DataObjectSink sinkTask = new DataObjectSink();
// computeGraphBuilder.addSource("datapointsource", sourceTask, parallelism);
// ComputeConnection firstGraphComputeConnection = computeGraphBuilder.addSink(
// "datapointsink", sinkTask, parallelism);
// firstGraphComputeConnection.direct("datapointsource",
// Context.TWISTER2_DIRECT_EDGE, DataType.OBJECT);
// computeGraphBuilder.setMode(OperationMode.BATCH);
//
// ComputeGraph datapointsTaskGraph = computeGraphBuilder.build();
// ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
// taskExecutor.execute(datapointsTaskGraph, firstGraphExecutionPlan);
// DataObject<Object> dataPointsObject = taskExecutor.getOutput(
// datapointsTaskGraph, firstGraphExecutionPlan, "datapointsink");
// LOG.info("Total Partitions : " + dataPointsObject.getPartitions().length);
/*
* Second Task
* */
DataSourceTask kMeansSourceTask = new DataSourceTask();
SimpleDataAllReduceTask kMeansAllReduceTask = new SimpleDataAllReduceTask();
computeGraphBuilder.addSource("kmeanssource", kMeansSourceTask, parallelism);
ComputeConnection computeConnection = computeGraphBuilder.addCompute("kmeanssink", kMeansAllReduceTask, parallelism);
computeConnection.allreduce("kmeanssource").viaEdge("all-reduce").withReductionFunction(new SimpleDataAggregator()).withDataType(MessageTypes.OBJECT);
computeGraphBuilder.setMode(OperationMode.BATCH);
ComputeGraph simpleTaskGraph = computeGraphBuilder.build();
ExecutionPlan plan = taskExecutor.plan(simpleTaskGraph);
// taskExecutor.addInput(
// simpleTaskGraph, plan, "kmeanssource", "points", dataPointsObject);
taskExecutor.execute(simpleTaskGraph, plan);
DataObject<double[][]> dataSet = taskExecutor.getOutput(simpleTaskGraph, plan, "kmeanssink");
// DataObject<Object> dataSet = taskExecutor.getOutput(simpleTaskGraph, plan, "kmeanssink");
// DataPartition<Object> values = dataSet.getPartitions()[0];
// Object lastObject = values.getConsumer().next();
// LOG.info(String.format("Last Object : %s", lastObject.getClass().getGraphName()));
}
Aggregations