Examples with ComputeGraphBuilder edu.iu.dsc.tws.task.impl.ComputeGraphBuilder used on opensource projects

Example 51 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class MultiStageGraph method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    GeneratorTask g = new GeneratorTask();
    ReduceTask rt = new ReduceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    builder.addSource("source", g, 4);
    ComputeConnection pc = builder.addCompute("compute", r, 4);
    pc.partition("source").viaEdge("partition-edge").withDataType(MessageTypes.OBJECT);
    ComputeConnection rc = builder.addCompute("sink", rt, 1);
    rc.reduce("compute").viaEdge("compute-edge").withReductionFunction((object1, object2) -> object1);
    builder.setMode(OperationMode.STREAMING);
    ComputeGraph graph = builder.build();
    graph.setGraphName("MultiTaskGraph");
    cEnv.getTaskExecutor().execute(graph, cEnv.getTaskExecutor().plan(graph));
}

Example 52 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class WordCount method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    // create source and aggregator
    WordSource source = new WordSource();
    WordAggregator counter = new WordAggregator();
    // build the graph
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    builder.addSource("word-source", source, 4);
    builder.addCompute("word-aggregator", counter, 4).partition("word-source").viaEdge(EDGE).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.STREAMING);
    // build the graph
    ComputeGraph graph = builder.build();
    // execute graph
    cEnv.getTaskExecutor().execute(graph);
}

Example 53 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class KMeansConnectedDataflowExample method generateData.

private static DataFlowGraph generateData(Config config, String dataDirectory, String centroidDirectory, int dimension, int dsize, int csize, int workers, int parallel, DataFlowJobConfig jobConfig) {
    DataGeneratorSource dataGeneratorSource = new DataGeneratorSource(Context.TWISTER2_DIRECT_EDGE, dsize, csize, dimension, dataDirectory, centroidDirectory);
    DataGeneratorSink dataGeneratorSink = new DataGeneratorSink();
    ComputeGraphBuilder dataGenerationGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    dataGenerationGraphBuilder.setTaskGraphName("DataGenerator");
    dataGenerationGraphBuilder.addSource("datageneratorsource", dataGeneratorSource, parallel);
    ComputeConnection dataObjectComputeConnection = dataGenerationGraphBuilder.addCompute("datageneratorsink", dataGeneratorSink, parallel);
    dataObjectComputeConnection.direct("datageneratorsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    dataGenerationGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph dataObjectTaskGraph = dataGenerationGraphBuilder.build();
    dataGenerationGraphBuilder.setTaskGraphName("datageneratorTG");
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("datageneratorTG", dataObjectTaskGraph).setWorkers(workers).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 54 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class KMeansConnectedDataflowExample method generateSecondJob.

private static DataFlowGraph generateSecondJob(Config config, int parallelismValue, String centroidDirectory, int dimension, int csize, int instances, DataFlowJobConfig jobConfig) {
    DataFileReplicatedReadSource dataFileReplicatedReadSource = new DataFileReplicatedReadSource(Context.TWISTER2_DIRECT_EDGE, centroidDirectory);
    KMeansDataObjectCompute centroidObjectCompute = new KMeansDataObjectCompute(Context.TWISTER2_DIRECT_EDGE, csize, dimension);
    KMeansDataObjectDirectSink centroidObjectSink = new KMeansDataObjectDirectSink("centroids");
    ComputeGraphBuilder centroidsComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    // Add source, compute, and sink tasks to the task graph builder for the second task graph
    centroidsComputeGraphBuilder.addSource("centroidsource", dataFileReplicatedReadSource, parallelismValue);
    ComputeConnection centroidComputeConnection = centroidsComputeGraphBuilder.addCompute("centroidcompute", centroidObjectCompute, parallelismValue);
    ComputeConnection secondGraphComputeConnection = centroidsComputeGraphBuilder.addCompute("centroidsink", centroidObjectSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    centroidComputeConnection.direct("centroidsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    secondGraphComputeConnection.direct("centroidcompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    centroidsComputeGraphBuilder.setMode(OperationMode.BATCH);
    centroidsComputeGraphBuilder.setTaskGraphName("centroidTG");
    // Build the second taskgraph
    ComputeGraph secondGraph = centroidsComputeGraphBuilder.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("centroidTG", secondGraph).setWorkers(instances).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 55 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class PageRankWorker method buildComputationTG.

public static ComputeGraph buildComputationTG(int parallelismValue, Config conf) {
    PageRankSource pageRankSource = new PageRankSource();
    PageRankKeyedReduce pageRankKeyedReduce = new PageRankKeyedReduce();
    PagerankSink pagerankSink = new PagerankSink();
    ComputeGraphBuilder pagerankComputationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    pagerankComputationTaskGraphBuilder.addSource("pageranksource", pageRankSource, parallelismValue);
    ComputeConnection computeConnectionKeyedReduce = pagerankComputationTaskGraphBuilder.addCompute("pagerankcompute", pageRankKeyedReduce, parallelismValue);
    ComputeConnection computeConnectionAllReduce = pagerankComputationTaskGraphBuilder.addCompute("pageranksink", pagerankSink, parallelismValue);
    computeConnectionKeyedReduce.keyedReduce("pageranksource").viaEdge("keyedreduce").withReductionFunction(new ReduceFn(Op.SUM, MessageTypes.DOUBLE_ARRAY)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.DOUBLE_ARRAY);
    computeConnectionAllReduce.allreduce("pagerankcompute").viaEdge("all-reduce").withReductionFunction(new Aggregate()).withDataType(MessageTypes.OBJECT);
    pagerankComputationTaskGraphBuilder.setMode(OperationMode.BATCH);
    pagerankComputationTaskGraphBuilder.setTaskGraphName("buildComputationTG");
    return pagerankComputationTaskGraphBuilder.build();
}