Examples with ComputeConnection edu.iu.dsc.tws.task.impl.ComputeConnection used on opensource projects

Example 6 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class BasicComputation method buildGraphInitialaizationTG.

protected ComputeGraph buildGraphInitialaizationTG(String path, int dsize, int parallelismValue, Config conf, GraphInitialization graphInitialization) {
    GraphDataSource initialaizatioSource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, path, dsize);
    DataInitializationSinkTask dataInitializationSinkTask = new DataInitializationSinkTask("InitialValue");
    ComputeGraphBuilder initialationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    initialationTaskGraphBuilder.addSource("initialaizatioSource", initialaizatioSource, parallelismValue);
    ComputeConnection datapointComputeConnection = initialationTaskGraphBuilder.addCompute("graphInitializationCompute", graphInitialization, parallelismValue);
    ComputeConnection firstGraphComputeConnection = initialationTaskGraphBuilder.addCompute("GraphInitializationSink", dataInitializationSinkTask, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("initialaizatioSource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("graphInitializationCompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    initialationTaskGraphBuilder.setMode(OperationMode.BATCH);
    initialationTaskGraphBuilder.setTaskGraphName("GraphInitialValueTG");
    // Build the first taskgraph
    return initialationTaskGraphBuilder.build();
}

Example 7 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class BasicComputation method buildComputationTG.

protected ComputeGraph buildComputationTG(int parallelismValue, Config conf, SourceTask sourceTask, ComputeTask computeTask, ReductionFunction reductionFunction, PrimitiveMessageTypes messageType) {
    SinkTask sinkTask = new SinkTask();
    ComputeGraphBuilder computationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    computationTaskGraphBuilder.addSource("source", sourceTask, parallelismValue);
    ComputeConnection computeConnectionKeyedReduce = computationTaskGraphBuilder.addCompute("compute", computeTask, parallelismValue);
    ComputeConnection computeConnectionAllReduce = computationTaskGraphBuilder.addCompute("sink", sinkTask, parallelismValue);
    if (reductionFunction == null) {
        computeConnectionKeyedReduce.keyedReduce("source").viaEdge("keyedreduce").withReductionFunction(new ReduceFn(Op.SUM, messageType)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.DOUBLE_ARRAY);
    } else {
        computeConnectionKeyedReduce.keyedReduce("source").viaEdge("keyedreduce").withReductionFunction(reductionFunction).withKeyType(MessageTypes.OBJECT).withDataType(messageType);
    }
    computeConnectionAllReduce.allreduce("compute").viaEdge("all-reduce").withReductionFunction(new Aggregate()).withDataType(MessageTypes.OBJECT);
    computationTaskGraphBuilder.setMode(OperationMode.BATCH);
    computationTaskGraphBuilder.setTaskGraphName("buildComputationTG");
    return computationTaskGraphBuilder.build();
}

Example 8 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class BasicComputation method buildGraphPartitionTG.

protected ComputeGraph buildGraphPartitionTG(String path, int dsize, int parallelismValue, Config conf, GraphPartiton graphPartiton) {
    GraphDataSource dataObjectSource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, path, dsize);
    DataSinkTask dataSinkTask = new DataSinkTask("PartitionSink");
    ComputeGraphBuilder graphPartitionTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    graphPartitionTaskGraphBuilder.addSource("Graphdatasource", dataObjectSource, parallelismValue);
    ComputeConnection datapointComputeConnection1 = graphPartitionTaskGraphBuilder.addCompute("Graphdatacompute", graphPartiton, parallelismValue);
    ComputeConnection datapointComputeConnection2 = graphPartitionTaskGraphBuilder.addCompute("GraphPartitionSink", dataSinkTask, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection1.direct("Graphdatasource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointComputeConnection2.direct("Graphdatacompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    graphPartitionTaskGraphBuilder.setMode(OperationMode.BATCH);
    graphPartitionTaskGraphBuilder.setTaskGraphName("datapointsTG");
    // Build the first taskgraph
    return graphPartitionTaskGraphBuilder.build();
}

Example 9 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class PageRankWorker method buildGraphInitialValueTG.

public static ComputeGraph buildGraphInitialValueTG(String dataDirectory, int dsize, int parallelismValue, Config conf) {
    GraphDataSource pageRankValueHolder = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory, dsize);
    PageRankValueHolderCompute pageRankValueHolderCompute = new PageRankValueHolderCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue);
    PageRankValueHolderSink pageRankValueHolderSink = new PageRankValueHolderSink("InitialValue");
    ComputeGraphBuilder pagerankInitialationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    pagerankInitialationTaskGraphBuilder.addSource("pageRankValueHolder", pageRankValueHolder, parallelismValue);
    ComputeConnection datapointComputeConnection = pagerankInitialationTaskGraphBuilder.addCompute("pageRankValueHolderCompute", pageRankValueHolderCompute, parallelismValue);
    ComputeConnection firstGraphComputeConnection = pagerankInitialationTaskGraphBuilder.addCompute("pageRankValueHolderSink", pageRankValueHolderSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("pageRankValueHolder").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("pageRankValueHolderCompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    pagerankInitialationTaskGraphBuilder.setMode(OperationMode.BATCH);
    pagerankInitialationTaskGraphBuilder.setTaskGraphName("GraphInitialValueTG");
    // Build the first taskgraph
    return pagerankInitialationTaskGraphBuilder.build();
}

Example 10 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class PageRankWorker method buildDataPointsTG.

public static ComputeGraph buildDataPointsTG(String dataDirectory, int dsize, int parallelismValue, Config conf) {
    GraphDataSource dataObjectSource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory, dsize);
    DataObjectCompute dataObjectCompute = new DataObjectCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue);
    DataObjectSink dataObjectSink = new DataObjectSink("Partitionsink");
    ComputeGraphBuilder datapointsTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    datapointsTaskGraphBuilder.addSource("Graphdatasource", dataObjectSource, parallelismValue);
    ComputeConnection datapointComputeConnection = datapointsTaskGraphBuilder.addCompute("Graphdatacompute", dataObjectCompute, parallelismValue);
    ComputeConnection firstGraphComputeConnection = datapointsTaskGraphBuilder.addCompute("Graphdatasink", dataObjectSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("Graphdatasource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("Graphdatacompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointsTaskGraphBuilder.setMode(OperationMode.BATCH);
    datapointsTaskGraphBuilder.setTaskGraphName("datapointsTG");
    // Build the first taskgraph
    return datapointsTaskGraphBuilder.build();
}