Examples with ComputeGraphBuilder edu.iu.dsc.tws.task.impl.ComputeGraphBuilder used on opensource projects

Example 1 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class SourceTaskDataLoader method execute.

@Override
public void execute() {
    getParams();
    /*
     * First data is loaded from files
     * */
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    // DataObjectSource sourceTask = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE,
    // dataSource);
    // DataObjectSink sinkTask = new DataObjectSink();
    // computeGraphBuilder.addSource("datapointsource", sourceTask, parallelism);
    // ComputeConnection firstGraphComputeConnection = computeGraphBuilder.addSink(
    // "datapointsink", sinkTask, parallelism);
    // firstGraphComputeConnection.direct("datapointsource",
    // Context.TWISTER2_DIRECT_EDGE, DataType.OBJECT);
    // computeGraphBuilder.setMode(OperationMode.BATCH);
    // 
    // ComputeGraph datapointsTaskGraph = computeGraphBuilder.build();
    // ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
    // taskExecutor.execute(datapointsTaskGraph, firstGraphExecutionPlan);
    // DataObject<Object> dataPointsObject = taskExecutor.getOutput(
    // datapointsTaskGraph, firstGraphExecutionPlan, "datapointsink");
    // LOG.info("Total Partitions : " + dataPointsObject.getPartitions().length);
    /*
     * Second Task
     * */
    DataSourceTask kMeansSourceTask = new DataSourceTask();
    SimpleDataAllReduceTask kMeansAllReduceTask = new SimpleDataAllReduceTask();
    computeGraphBuilder.addSource("kmeanssource", kMeansSourceTask, parallelism);
    ComputeConnection computeConnection = computeGraphBuilder.addCompute("kmeanssink", kMeansAllReduceTask, parallelism);
    computeConnection.allreduce("kmeanssource").viaEdge("all-reduce").withReductionFunction(new SimpleDataAggregator()).withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph simpleTaskGraph = computeGraphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(simpleTaskGraph);
    // taskExecutor.addInput(
    // simpleTaskGraph, plan, "kmeanssource", "points", dataPointsObject);
    taskExecutor.execute(simpleTaskGraph, plan);
    DataObject<double[][]> dataSet = taskExecutor.getOutput(simpleTaskGraph, plan, "kmeanssink");
// DataObject<Object> dataSet = taskExecutor.getOutput(simpleTaskGraph, plan, "kmeanssink");
// DataPartition<Object> values = dataSet.getPartitions()[0];
// Object lastObject = values.getConsumer().next();
// LOG.info(String.format("Last Object : %s", lastObject.getClass().getGraphName()));
}

Example 2 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class TaskWorkerDataLoader method execute.

@Override
public void execute() {
    getParams();
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    DataObjectSource sourceTask = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE, dataSource);
    DataObjectSink sinkTask = new DataObjectSink();
    computeGraphBuilder.addSource("datapointsource", sourceTask, parallelism);
    ComputeConnection firstGraphComputeConnection = computeGraphBuilder.addCompute("datapointsink", sinkTask, parallelism);
    firstGraphComputeConnection.direct("datapointsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph datapointsTaskGraph = computeGraphBuilder.build();
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
    taskExecutor.execute(datapointsTaskGraph, firstGraphExecutionPlan);
    DataObject<Object> dataPointsObject = taskExecutor.getOutput(datapointsTaskGraph, firstGraphExecutionPlan, "datapointsink");
    LOG.info("Total Partitions : " + dataPointsObject.getPartitions().length);
    showAllUnits(dataPointsObject);
}

Example 3 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class BasicComputation method buildGraphInitialaizationTG.

protected ComputeGraph buildGraphInitialaizationTG(String path, int dsize, int parallelismValue, Config conf, GraphInitialization graphInitialization) {
    GraphDataSource initialaizatioSource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, path, dsize);
    DataInitializationSinkTask dataInitializationSinkTask = new DataInitializationSinkTask("InitialValue");
    ComputeGraphBuilder initialationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    initialationTaskGraphBuilder.addSource("initialaizatioSource", initialaizatioSource, parallelismValue);
    ComputeConnection datapointComputeConnection = initialationTaskGraphBuilder.addCompute("graphInitializationCompute", graphInitialization, parallelismValue);
    ComputeConnection firstGraphComputeConnection = initialationTaskGraphBuilder.addCompute("GraphInitializationSink", dataInitializationSinkTask, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("initialaizatioSource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("graphInitializationCompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    initialationTaskGraphBuilder.setMode(OperationMode.BATCH);
    initialationTaskGraphBuilder.setTaskGraphName("GraphInitialValueTG");
    // Build the first taskgraph
    return initialationTaskGraphBuilder.build();
}

Example 4 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class BasicComputation method buildComputationTG.

protected ComputeGraph buildComputationTG(int parallelismValue, Config conf, SourceTask sourceTask, ComputeTask computeTask, ReductionFunction reductionFunction, PrimitiveMessageTypes messageType) {
    SinkTask sinkTask = new SinkTask();
    ComputeGraphBuilder computationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    computationTaskGraphBuilder.addSource("source", sourceTask, parallelismValue);
    ComputeConnection computeConnectionKeyedReduce = computationTaskGraphBuilder.addCompute("compute", computeTask, parallelismValue);
    ComputeConnection computeConnectionAllReduce = computationTaskGraphBuilder.addCompute("sink", sinkTask, parallelismValue);
    if (reductionFunction == null) {
        computeConnectionKeyedReduce.keyedReduce("source").viaEdge("keyedreduce").withReductionFunction(new ReduceFn(Op.SUM, messageType)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.DOUBLE_ARRAY);
    } else {
        computeConnectionKeyedReduce.keyedReduce("source").viaEdge("keyedreduce").withReductionFunction(reductionFunction).withKeyType(MessageTypes.OBJECT).withDataType(messageType);
    }
    computeConnectionAllReduce.allreduce("compute").viaEdge("all-reduce").withReductionFunction(new Aggregate()).withDataType(MessageTypes.OBJECT);
    computationTaskGraphBuilder.setMode(OperationMode.BATCH);
    computationTaskGraphBuilder.setTaskGraphName("buildComputationTG");
    return computationTaskGraphBuilder.build();
}

Example 5 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class BasicComputation method buildGraphPartitionTG.

protected ComputeGraph buildGraphPartitionTG(String path, int dsize, int parallelismValue, Config conf, GraphPartiton graphPartiton) {
    GraphDataSource dataObjectSource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, path, dsize);
    DataSinkTask dataSinkTask = new DataSinkTask("PartitionSink");
    ComputeGraphBuilder graphPartitionTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    graphPartitionTaskGraphBuilder.addSource("Graphdatasource", dataObjectSource, parallelismValue);
    ComputeConnection datapointComputeConnection1 = graphPartitionTaskGraphBuilder.addCompute("Graphdatacompute", graphPartiton, parallelismValue);
    ComputeConnection datapointComputeConnection2 = graphPartitionTaskGraphBuilder.addCompute("GraphPartitionSink", dataSinkTask, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection1.direct("Graphdatasource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointComputeConnection2.direct("Graphdatacompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    graphPartitionTaskGraphBuilder.setMode(OperationMode.BATCH);
    graphPartitionTaskGraphBuilder.setTaskGraphName("datapointsTG");
    // Build the first taskgraph
    return graphPartitionTaskGraphBuilder.build();
}