Examples with ComputeGraphBuilder edu.iu.dsc.tws.task.impl.ComputeGraphBuilder used on opensource projects

Example 56 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class SingleSourceShortestPathWorker method buildDataPointsTG.

public static ComputeGraph buildDataPointsTG(String dataDirectory, int dsize, int parallelismValue, String soruceVertex, Config conf) {
    GraphDataSource dataObjectSource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory, dsize);
    GraphDataCompute graphDataCompute = new GraphDataCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue, soruceVertex);
    GraphDataSink graphDataSink = new GraphDataSink("PartitionSink");
    ComputeGraphBuilder datapointsTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    datapointsTaskGraphBuilder.addSource("Graphdatasource", dataObjectSource, parallelismValue);
    ComputeConnection datapointComputeConnection = datapointsTaskGraphBuilder.addCompute("Graphdatacompute", graphDataCompute, parallelismValue);
    ComputeConnection firstGraphComputeConnection = datapointsTaskGraphBuilder.addCompute("Graphdatasink", graphDataSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("Graphdatasource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("Graphdatacompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointsTaskGraphBuilder.setMode(OperationMode.BATCH);
    datapointsTaskGraphBuilder.setTaskGraphName("datapointsTG");
    // Build the first taskgraph
    return datapointsTaskGraphBuilder.build();
}

Example 57 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class ComputeEnvironment method newTaskGraph.

/**
 * Create a new task graph builder with a name
 *
 * @param operationMode operation node
 * @param name name of the graph
 * @return the graph builder
 */
public ComputeGraphBuilder newTaskGraph(OperationMode operationMode, String name) {
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(workerEnvironment.getConfig());
    computeGraphBuilder.setMode(operationMode);
    computeGraphBuilder.setTaskGraphName(name);
    return computeGraphBuilder;
}

Example 58 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class ConstraintTaskExample method buildFirstGraph.

private ComputeGraph buildFirstGraph(int parallelism, Config conf, String dataInput, int dataSize, int dimension, String inputKey, String constraint) {
    FirstSourceTask sourceTask = new FirstSourceTask(dataInput, dataSize);
    FirstSinkTask sinkTask = new FirstSinkTask(dimension, inputKey);
    ComputeGraphBuilder firstGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    firstGraphBuilder.addSource("firstsource", sourceTask, parallelism);
    ComputeConnection computeConnection = firstGraphBuilder.addCompute("firstsink", sinkTask, parallelism);
    computeConnection.direct("firstsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphBuilder.setMode(OperationMode.BATCH);
    firstGraphBuilder.setTaskGraphName("firstTG");
    firstGraphBuilder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, constraint);
    return firstGraphBuilder.build();
}

Example 59 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class DataflowAddNodeExperiment method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    SourceTask sourceTask = new SourceTask();
    ReduceTask reduceTask = new ReduceTask();
    FirstComputeTask firstComputeTask = new FirstComputeTask();
    SecondComputeTask secondComputeTask = new SecondComputeTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    DataflowJobParameters dataflowJobParameters = DataflowJobParameters.build(config);
    int parallel = dataflowJobParameters.getParallelismValue();
    int iter = dataflowJobParameters.getIterations();
    builder.addSource("source", sourceTask, parallel);
    ComputeConnection computeConnection = builder.addCompute("firstcompute", firstComputeTask, parallel);
    ComputeConnection computeConnection1 = builder.addCompute("secondcompute", secondComputeTask, parallel);
    ComputeConnection rc = builder.addCompute("sink", reduceTask, parallel);
    computeConnection.direct("source").viaEdge("fdirect").withDataType(MessageTypes.OBJECT);
    computeConnection1.direct("firstcompute").viaEdge("sdirect").withDataType(MessageTypes.OBJECT);
    rc.allreduce("secondcompute").viaEdge("all-reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    // ExecutionPlan plan = taskExecutor.plan(graph);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < iter; i++) {
        // taskExecutor.execute(graph, plan);
        LOG.info("Completed Iteration:" + i);
    }
    long stopTime = System.currentTimeMillis();
    long executionTime = stopTime - startTime;
    LOG.info("Total Execution Time to complete Dataflow Additional Node Experiment" + "\t" + executionTime + "(in milliseconds)");
}

Example 60 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerExample method buildFirstGraph.

private static ComputeGraph buildFirstGraph(int parallelism, Config conf) {
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    FirstSourceTask sourceTask = new FirstSourceTask();
    FirstComputeTask computeTask = new FirstComputeTask();
    FirstSinkTask sinkTask = new FirstSinkTask("firstgraphpoints");
    ComputeGraphBuilder firstGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    firstGraphBuilder.addSource("firstsource", sourceTask, parallelism);
    ComputeConnection computeConnection = firstGraphBuilder.addCompute("firstcompute", computeTask, parallelism);
    ComputeConnection sinkConnection = firstGraphBuilder.addCompute("firstsink", sinkTask, parallelism);
    // Creating the communication edges between the tasks for the second task graph
    computeConnection.direct("firstsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    sinkConnection.direct("firstcompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphBuilder.setMode(OperationMode.BATCH);
    firstGraphBuilder.setTaskGraphName("firstTG");
    return firstGraphBuilder.build();
}