Examples with ComputeConnection edu.iu.dsc.tws.task.impl.ComputeConnection used on opensource projects

Example 11 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class SingleSourceShortestPathWorker method buildComputationSsspTG.

public static ComputeGraph buildComputationSsspTG(int parallelismValue, Config conf) {
    SsspSource ssspSource = new SsspSource();
    SsspKeyedReduce ssspKeyedReduce = new SsspKeyedReduce();
    SsspSink ssspSink = new SsspSink();
    ComputeGraphBuilder ssspTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    ssspTaskGraphBuilder.addSource("ssspSource", ssspSource, parallelismValue);
    ComputeConnection computeConnectionKeyedReduce = ssspTaskGraphBuilder.addCompute("ssspKeyedReduce", ssspKeyedReduce, parallelismValue);
    ComputeConnection computeConnectionAllReduce = ssspTaskGraphBuilder.addCompute("ssspSink", ssspSink, parallelismValue);
    computeConnectionKeyedReduce.keyedReduce("ssspSource").viaEdge("keyedreduce").withReductionFunction(new Keyedreducefun()).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.INTEGER_ARRAY);
    computeConnectionAllReduce.allreduce("ssspKeyedReduce").viaEdge("all-reduce").withReductionFunction(new AggregateFn()).withDataType(MessageTypes.OBJECT);
    ssspTaskGraphBuilder.setMode(OperationMode.BATCH);
    ssspTaskGraphBuilder.setTaskGraphName("ComputationSsspTG");
    return ssspTaskGraphBuilder.build();
}

Example 12 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class SingleSourceShortestPathWorker method buildSsspInitialTG.

public static ComputeGraph buildSsspInitialTG(String dataDirectory, int dsize, int parallelismValue, String soruceVertex, Config conf) {
    // the second task graph for assign initial pagerank values for vertex.
    GraphDataSource ssspInitialDatasource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory, dsize);
    SsspInitialCompute ssspInitialCompute = new SsspInitialCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue, soruceVertex);
    SsspInitialSink ssspInitialSink = new SsspInitialSink("InitialValue");
    ComputeGraphBuilder datapointsTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    datapointsTaskGraphBuilder.addSource("ssspInitialDatasource", ssspInitialDatasource, parallelismValue);
    ComputeConnection datapointComputeConnection = datapointsTaskGraphBuilder.addCompute("ssspInitialCompute", ssspInitialCompute, parallelismValue);
    ComputeConnection firstGraphComputeConnection = datapointsTaskGraphBuilder.addCompute("ssspInitialSink", ssspInitialSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("ssspInitialDatasource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("ssspInitialCompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointsTaskGraphBuilder.setMode(OperationMode.BATCH);
    datapointsTaskGraphBuilder.setTaskGraphName("SsspInitialTG");
    // Build the first taskgraph
    return datapointsTaskGraphBuilder.build();
}

Example 13 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method createGraph.

private ComputeGraph createGraph(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection c = builder.addCompute("sink", testSink, parallel);
    c.reduce("source").viaEdge("edge").withOperation(Op.SUM, MessageTypes.INTEGER_ARRAY);
    builder.setMode(OperationMode.STREAMING);
    return builder.build();
}

Example 14 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method createGraphWithComputeTaskAndConstraints.

private ComputeGraph createGraphWithComputeTaskAndConstraints(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestCompute testCompute = new TaskSchedulerClassTest.TestCompute();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    computeGraphBuilder.addSource("source", testSource, parallel);
    ComputeConnection computeConnection = computeGraphBuilder.addCompute("compute", testCompute, parallel);
    ComputeConnection sinkComputeConnection = computeGraphBuilder.addCompute("sink", testSink, parallel);
    computeConnection.direct("source").viaEdge("cdirect-edge").withDataType(MessageTypes.OBJECT);
    sinkComputeConnection.direct("compute").viaEdge("sdirect-edge").withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.STREAMING);
    computeGraphBuilder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "24");
    ComputeGraph taskGraph = computeGraphBuilder.build();
    return taskGraph;
}

Example 15 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method createGraphWithComputeTaskAndConstraints.

private ComputeGraph createGraphWithComputeTaskAndConstraints(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestCompute testCompute = new TaskSchedulerClassTest.TestCompute();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection computeConnection = builder.addCompute("compute", testCompute, parallel);
    ComputeConnection sinkConnection = builder.addCompute("sink", testSink, parallel);
    computeConnection.direct("source").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    sinkConnection.direct("compute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    builder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "2");
    ComputeGraph graph = builder.build();
    return graph;
}