Examples with ComputeGraphBuilder edu.iu.dsc.tws.task.impl.ComputeGraphBuilder used on opensource projects

Example 61 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class DataflowNodeExperiment method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    SourceTask sourceTask = new SourceTask();
    ReduceTask reduceTask = new ReduceTask();
    ComputeTask computeTask = new ComputeTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    DataflowJobParameters dataflowJobParameters = DataflowJobParameters.build(config);
    int parallel = dataflowJobParameters.getParallelismValue();
    int iter = dataflowJobParameters.getIterations();
    builder.addSource("source", sourceTask, parallel);
    ComputeConnection computeConnection = builder.addCompute("compute", computeTask, parallel);
    ComputeConnection rc = builder.addCompute("sink", reduceTask, parallel);
    computeConnection.direct("source").viaEdge("direct").withDataType(MessageTypes.OBJECT);
    rc.allreduce("compute").viaEdge("all-reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < iter; i++) {
        taskExecutor.execute(graph, plan);
        LOG.info("Completed Iteration:" + i);
    }
    long stopTime = System.currentTimeMillis();
    long executionTime = stopTime - startTime;
    LOG.info("Total Execution Time to Complete Dataflow Node Experiment" + "\t" + executionTime + "(in milliseconds)");
}

Example 62 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class KafkaExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    ComputeGraphBuilder graphBuilder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    graphBuilder.setMode(OperationMode.STREAMING);
    graphBuilder.addSource("ksource", new KSource(), 2);
    graphBuilder.addCompute("sink", new KSink(), 2).direct("ksource").viaEdge("edge");
    cEnv.buildAndExecute(graphBuilder);
}

Example 63 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class WordCountJob method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    // source and aggregator
    WordSource source = new WordSource();
    WordAggregator counter = new WordAggregator();
    // build the task graph
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    builder.addSource("word-source", source, 4);
    builder.addCompute("word-aggregator", counter, 4).keyedReduce("word-source").viaEdge(EDGE).withReductionFunction(new ReduceFn(Op.SUM, MessageTypes.INTEGER_ARRAY)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.INTEGER_ARRAY);
    builder.setMode(OperationMode.BATCH);
    // execute the graph
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = cEnv.getTaskExecutor().plan(graph);
    cEnv.getTaskExecutor().execute(graph, plan);
}

Example 64 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class TwoDataFlowsExample method runFirstJob.

private static void runFirstJob(Config config, CDFWEnv cdfwEnv, int parallelism, DataFlowJobConfig jobConfig) {
    FirstSourceTask firstSourceTask = new FirstSourceTask();
    ConnectedSink connectedSink = new ConnectedSink("first_out");
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source1", firstSourceTask, parallelism);
    ComputeConnection partitionConnection = graphBuilderX.addCompute("sink1", connectedSink, parallelism);
    partitionConnection.partition("source1").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("first_graph", batchGraph).setWorkers(4).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    cdfwEnv.executeDataFlowGraph(job);
}

Example 65 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class TwoDataFlowsExample method runSecondJob.

private static void runSecondJob(Config config, CDFWEnv cdfwEnv, int parallelism, DataFlowJobConfig jobConfig) {
    ConnectedSource connectedSource = new ConnectedSource("reduce", "first_out");
    ConnectedSink connectedSink = new ConnectedSink();
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source2", connectedSource, parallelism);
    ComputeConnection reduceConn = graphBuilderX.addCompute("sink2", connectedSink, 1);
    reduceConn.reduce("source2").viaEdge("reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("second_graph", batchGraph).setWorkers(4).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    cdfwEnv.executeDataFlowGraph(job);
}