Examples with ComputeConnection edu.iu.dsc.tws.task.impl.ComputeConnection used on opensource projects

Example 61 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class DataflowAddNodeExperiment method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    SourceTask sourceTask = new SourceTask();
    ReduceTask reduceTask = new ReduceTask();
    FirstComputeTask firstComputeTask = new FirstComputeTask();
    SecondComputeTask secondComputeTask = new SecondComputeTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    DataflowJobParameters dataflowJobParameters = DataflowJobParameters.build(config);
    int parallel = dataflowJobParameters.getParallelismValue();
    int iter = dataflowJobParameters.getIterations();
    builder.addSource("source", sourceTask, parallel);
    ComputeConnection computeConnection = builder.addCompute("firstcompute", firstComputeTask, parallel);
    ComputeConnection computeConnection1 = builder.addCompute("secondcompute", secondComputeTask, parallel);
    ComputeConnection rc = builder.addCompute("sink", reduceTask, parallel);
    computeConnection.direct("source").viaEdge("fdirect").withDataType(MessageTypes.OBJECT);
    computeConnection1.direct("firstcompute").viaEdge("sdirect").withDataType(MessageTypes.OBJECT);
    rc.allreduce("secondcompute").viaEdge("all-reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    // ExecutionPlan plan = taskExecutor.plan(graph);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < iter; i++) {
        // taskExecutor.execute(graph, plan);
        LOG.info("Completed Iteration:" + i);
    }
    long stopTime = System.currentTimeMillis();
    long executionTime = stopTime - startTime;
    LOG.info("Total Execution Time to complete Dataflow Additional Node Experiment" + "\t" + executionTime + "(in milliseconds)");
}

Example 62 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerExample method buildFirstGraph.

private static ComputeGraph buildFirstGraph(int parallelism, Config conf) {
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    FirstSourceTask sourceTask = new FirstSourceTask();
    FirstComputeTask computeTask = new FirstComputeTask();
    FirstSinkTask sinkTask = new FirstSinkTask("firstgraphpoints");
    ComputeGraphBuilder firstGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    firstGraphBuilder.addSource("firstsource", sourceTask, parallelism);
    ComputeConnection computeConnection = firstGraphBuilder.addCompute("firstcompute", computeTask, parallelism);
    ComputeConnection sinkConnection = firstGraphBuilder.addCompute("firstsink", sinkTask, parallelism);
    // Creating the communication edges between the tasks for the second task graph
    computeConnection.direct("firstsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    sinkConnection.direct("firstcompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphBuilder.setMode(OperationMode.BATCH);
    firstGraphBuilder.setTaskGraphName("firstTG");
    return firstGraphBuilder.build();
}

Example 63 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class DataflowNodeExperiment method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    SourceTask sourceTask = new SourceTask();
    ReduceTask reduceTask = new ReduceTask();
    ComputeTask computeTask = new ComputeTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    DataflowJobParameters dataflowJobParameters = DataflowJobParameters.build(config);
    int parallel = dataflowJobParameters.getParallelismValue();
    int iter = dataflowJobParameters.getIterations();
    builder.addSource("source", sourceTask, parallel);
    ComputeConnection computeConnection = builder.addCompute("compute", computeTask, parallel);
    ComputeConnection rc = builder.addCompute("sink", reduceTask, parallel);
    computeConnection.direct("source").viaEdge("direct").withDataType(MessageTypes.OBJECT);
    rc.allreduce("compute").viaEdge("all-reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < iter; i++) {
        taskExecutor.execute(graph, plan);
        LOG.info("Completed Iteration:" + i);
    }
    long stopTime = System.currentTimeMillis();
    long executionTime = stopTime - startTime;
    LOG.info("Total Execution Time to Complete Dataflow Node Experiment" + "\t" + executionTime + "(in milliseconds)");
}

Example 64 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class TwoDataFlowsExample method runFirstJob.

private static void runFirstJob(Config config, CDFWEnv cdfwEnv, int parallelism, DataFlowJobConfig jobConfig) {
    FirstSourceTask firstSourceTask = new FirstSourceTask();
    ConnectedSink connectedSink = new ConnectedSink("first_out");
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source1", firstSourceTask, parallelism);
    ComputeConnection partitionConnection = graphBuilderX.addCompute("sink1", connectedSink, parallelism);
    partitionConnection.partition("source1").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("first_graph", batchGraph).setWorkers(4).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    cdfwEnv.executeDataFlowGraph(job);
}

Example 65 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class TwoDataFlowsExample method runSecondJob.

private static void runSecondJob(Config config, CDFWEnv cdfwEnv, int parallelism, DataFlowJobConfig jobConfig) {
    ConnectedSource connectedSource = new ConnectedSource("reduce", "first_out");
    ConnectedSink connectedSink = new ConnectedSink();
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source2", connectedSource, parallelism);
    ComputeConnection reduceConn = graphBuilderX.addCompute("sink2", connectedSink, 1);
    reduceConn.reduce("source2").viaEdge("reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("second_graph", batchGraph).setWorkers(4).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    cdfwEnv.executeDataFlowGraph(job);
}