Examples with DataFlowGraph edu.iu.dsc.tws.task.cdfw.DataFlowGraph used on opensource projects

Example 1 with DataFlowGraph

use of edu.iu.dsc.tws.task.cdfw.DataFlowGraph in project twister2 by DSC-SPIDAL.

the class BatchTSetCDFWEnvironment method executeBuildContext.

@Override
protected void executeBuildContext(BuildContext buildContext) {
    DataFlowJobConfig dafaFlowJobConfig = new DataFlowJobConfig();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("hello", buildContext.getComputeGraph()).setWorkers(2).addDataFlowJobConfig(dafaFlowJobConfig).setGraphType("non-iterative");
    cdfwEnv.executeDataFlowGraph(job);
}

Example 2 with DataFlowGraph

use of edu.iu.dsc.tws.task.cdfw.DataFlowGraph in project twister2 by DSC-SPIDAL.

the class ParallelDataFlowsExample method generateSecondJob.

private static DataFlowGraph generateSecondJob(Config config, int parallelismValue, int workers, DataFlowJobConfig jobConfig) {
    ConnectedSource connectedSource = new ConnectedSource("reduce", "first_out");
    ConnectedSink connectedSink = new ConnectedSink();
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source1", connectedSource, parallelismValue);
    ComputeConnection reduceConn = graphBuilderX.addCompute("sink1", connectedSink, 1);
    reduceConn.reduce("source1").viaEdge("reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("second_graph", batchGraph).setWorkers(workers).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 3 with DataFlowGraph

use of edu.iu.dsc.tws.task.cdfw.DataFlowGraph in project twister2 by DSC-SPIDAL.

the class ParallelDataFlowsExample method generateFirstJob.

private static DataFlowGraph generateFirstJob(Config config, int parallelismValue, int workers, DataFlowJobConfig jobConfig) {
    FirstSourceTask firstSourceTask = new FirstSourceTask();
    ConnectedSink connectedSink = new ConnectedSink("first_out");
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source1", firstSourceTask, parallelismValue);
    ComputeConnection partitionConnection = graphBuilderX.addCompute("sink1", connectedSink, parallelismValue);
    partitionConnection.partition("source1").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("first_graph", batchGraph).setWorkers(workers).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 4 with DataFlowGraph

use of edu.iu.dsc.tws.task.cdfw.DataFlowGraph in project twister2 by DSC-SPIDAL.

the class KMeansConnectedDataflowExample method generateThirdJob.

private static DataFlowGraph generateThirdJob(Config config, int parallelismValue, int instances, int iterations, int dimension, DataFlowJobConfig jobConfig) {
    KMeansSourceTask kMeansSourceTask = new KMeansSourceTask(dimension);
    KMeansAllReduceTask kMeansAllReduceTask = new KMeansAllReduceTask();
    ComputeGraphBuilder kmeansComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    // Add source, and sink tasks to the task graph builder for the third task graph
    kmeansComputeGraphBuilder.addSource("kmeanssource", kMeansSourceTask, parallelismValue);
    ComputeConnection kMeanscomputeConnection = kmeansComputeGraphBuilder.addCompute("kmeanssink", kMeansAllReduceTask, parallelismValue);
    // Creating the communication edges between the tasks for the third task graph
    kMeanscomputeConnection.allreduce("kmeanssource").viaEdge("all-reduce").withReductionFunction(new CentroidAggregator()).withDataType(MessageTypes.OBJECT);
    kmeansComputeGraphBuilder.setMode(OperationMode.BATCH);
    kmeansComputeGraphBuilder.setTaskGraphName("kmeansTG");
    ComputeGraph thirdGraph = kmeansComputeGraphBuilder.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("kmeansTG", thirdGraph).setWorkers(instances).addDataFlowJobConfig(jobConfig).setGraphType("iterative").setIterations(iterations);
    return job;
}

Example 5 with DataFlowGraph

use of edu.iu.dsc.tws.task.cdfw.DataFlowGraph in project twister2 by DSC-SPIDAL.

the class KMeansConnectedDataflowExample method generateFirstJob.

private static DataFlowGraph generateFirstJob(Config config, int parallelismValue, String dataDirectory, int dimension, int dsize, int instances, DataFlowJobConfig jobConfig) {
    DataObjectSource dataObjectSource = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory);
    KMeansDataObjectCompute dataObjectCompute = new KMeansDataObjectCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue, dimension);
    KMeansDataObjectDirectSink dataObjectSink = new KMeansDataObjectDirectSink("points");
    ComputeGraphBuilder datapointsComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    datapointsComputeGraphBuilder.addSource("datapointsource", dataObjectSource, parallelismValue);
    ComputeConnection datapointComputeConnection = datapointsComputeGraphBuilder.addCompute("datapointcompute", dataObjectCompute, parallelismValue);
    ComputeConnection firstGraphComputeConnection = datapointsComputeGraphBuilder.addCompute("datapointsink", dataObjectSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("datapointsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("datapointcompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointsComputeGraphBuilder.setMode(OperationMode.BATCH);
    datapointsComputeGraphBuilder.setTaskGraphName("datapointsTG");
    ComputeGraph firstGraph = datapointsComputeGraphBuilder.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("datapointsTG", firstGraph).setWorkers(instances).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}