Examples with ComputeGraph edu.iu.dsc.tws.api.compute.graph.ComputeGraph used on opensource projects

Example 56 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class MultiStageGraph method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    GeneratorTask g = new GeneratorTask();
    ReduceTask rt = new ReduceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    builder.addSource("source", g, 4);
    ComputeConnection pc = builder.addCompute("compute", r, 4);
    pc.partition("source").viaEdge("partition-edge").withDataType(MessageTypes.OBJECT);
    ComputeConnection rc = builder.addCompute("sink", rt, 1);
    rc.reduce("compute").viaEdge("compute-edge").withReductionFunction((object1, object2) -> object1);
    builder.setMode(OperationMode.STREAMING);
    ComputeGraph graph = builder.build();
    graph.setGraphName("MultiTaskGraph");
    cEnv.getTaskExecutor().execute(graph, cEnv.getTaskExecutor().plan(graph));
}

Example 57 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class WordCount method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    // create source and aggregator
    WordSource source = new WordSource();
    WordAggregator counter = new WordAggregator();
    // build the graph
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    builder.addSource("word-source", source, 4);
    builder.addCompute("word-aggregator", counter, 4).partition("word-source").viaEdge(EDGE).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.STREAMING);
    // build the graph
    ComputeGraph graph = builder.build();
    // execute graph
    cEnv.getTaskExecutor().execute(graph);
}

Example 58 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class ConstraintTaskExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    long startTime = System.currentTimeMillis();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    String dinput = String.valueOf(config.get(DataObjectConstants.DINPUT_DIRECTORY));
    int dimension = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.DIMENSIONS)));
    int parallelismValue = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.PARALLELISM_VALUE)));
    int dsize = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.DSIZE)));
    DataGenerator dataGenerator = new DataGenerator(config, workerId);
    dataGenerator.generate(new Path(dinput), dsize, dimension);
    ComputeGraph firstGraph = buildFirstGraph(parallelismValue, config, dinput, dsize, dimension, "firstgraphpoints", "1");
    ComputeGraph secondGraph = buildSecondGraph(parallelismValue, config, dimension, "firstgraphpoints", "1");
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(firstGraph);
    taskExecutor.execute(firstGraph, firstGraphExecutionPlan);
    DataObject<Object> firstGraphObject = taskExecutor.getOutput("firstsink");
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(secondGraph);
    taskExecutor.addInput("firstgraphpoints", firstGraphObject);
    taskExecutor.execute(secondGraph, secondGraphExecutionPlan);
    long endTime = System.currentTimeMillis();
    LOG.info("Total Execution Time: " + (endTime - startTime));
}

Example 59 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class DataflowAddNodeExperiment method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    SourceTask sourceTask = new SourceTask();
    ReduceTask reduceTask = new ReduceTask();
    FirstComputeTask firstComputeTask = new FirstComputeTask();
    SecondComputeTask secondComputeTask = new SecondComputeTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    DataflowJobParameters dataflowJobParameters = DataflowJobParameters.build(config);
    int parallel = dataflowJobParameters.getParallelismValue();
    int iter = dataflowJobParameters.getIterations();
    builder.addSource("source", sourceTask, parallel);
    ComputeConnection computeConnection = builder.addCompute("firstcompute", firstComputeTask, parallel);
    ComputeConnection computeConnection1 = builder.addCompute("secondcompute", secondComputeTask, parallel);
    ComputeConnection rc = builder.addCompute("sink", reduceTask, parallel);
    computeConnection.direct("source").viaEdge("fdirect").withDataType(MessageTypes.OBJECT);
    computeConnection1.direct("firstcompute").viaEdge("sdirect").withDataType(MessageTypes.OBJECT);
    rc.allreduce("secondcompute").viaEdge("all-reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    // ExecutionPlan plan = taskExecutor.plan(graph);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < iter; i++) {
        // taskExecutor.execute(graph, plan);
        LOG.info("Completed Iteration:" + i);
    }
    long stopTime = System.currentTimeMillis();
    long executionTime = stopTime - startTime;
    LOG.info("Total Execution Time to complete Dataflow Additional Node Experiment" + "\t" + executionTime + "(in milliseconds)");
}

Example 60 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class DataflowNodeExperiment method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    SourceTask sourceTask = new SourceTask();
    ReduceTask reduceTask = new ReduceTask();
    ComputeTask computeTask = new ComputeTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    DataflowJobParameters dataflowJobParameters = DataflowJobParameters.build(config);
    int parallel = dataflowJobParameters.getParallelismValue();
    int iter = dataflowJobParameters.getIterations();
    builder.addSource("source", sourceTask, parallel);
    ComputeConnection computeConnection = builder.addCompute("compute", computeTask, parallel);
    ComputeConnection rc = builder.addCompute("sink", reduceTask, parallel);
    computeConnection.direct("source").viaEdge("direct").withDataType(MessageTypes.OBJECT);
    rc.allreduce("compute").viaEdge("all-reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < iter; i++) {
        taskExecutor.execute(graph, plan);
        LOG.info("Completed Iteration:" + i);
    }
    long stopTime = System.currentTimeMillis();
    long executionTime = stopTime - startTime;
    LOG.info("Total Execution Time to Complete Dataflow Node Experiment" + "\t" + executionTime + "(in milliseconds)");
}