Examples with ReduceFn edu.iu.dsc.tws.task.impl.function.ReduceFn used on opensource projects

Example 1 with ReduceFn

use of edu.iu.dsc.tws.task.impl.function.ReduceFn in project twister2 by DSC-SPIDAL.

the class BasicComputation method buildComputationTG.

protected ComputeGraph buildComputationTG(int parallelismValue, Config conf, SourceTask sourceTask, ComputeTask computeTask, ReductionFunction reductionFunction, PrimitiveMessageTypes messageType) {
    SinkTask sinkTask = new SinkTask();
    ComputeGraphBuilder computationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    computationTaskGraphBuilder.addSource("source", sourceTask, parallelismValue);
    ComputeConnection computeConnectionKeyedReduce = computationTaskGraphBuilder.addCompute("compute", computeTask, parallelismValue);
    ComputeConnection computeConnectionAllReduce = computationTaskGraphBuilder.addCompute("sink", sinkTask, parallelismValue);
    if (reductionFunction == null) {
        computeConnectionKeyedReduce.keyedReduce("source").viaEdge("keyedreduce").withReductionFunction(new ReduceFn(Op.SUM, messageType)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.DOUBLE_ARRAY);
    } else {
        computeConnectionKeyedReduce.keyedReduce("source").viaEdge("keyedreduce").withReductionFunction(reductionFunction).withKeyType(MessageTypes.OBJECT).withDataType(messageType);
    }
    computeConnectionAllReduce.allreduce("compute").viaEdge("all-reduce").withReductionFunction(new Aggregate()).withDataType(MessageTypes.OBJECT);
    computationTaskGraphBuilder.setMode(OperationMode.BATCH);
    computationTaskGraphBuilder.setTaskGraphName("buildComputationTG");
    return computationTaskGraphBuilder.build();
}

Example 2 with ReduceFn

use of edu.iu.dsc.tws.task.impl.function.ReduceFn in project twister2 by DSC-SPIDAL.

the class PageRankWorker method buildComputationTG.

public static ComputeGraph buildComputationTG(int parallelismValue, Config conf) {
    PageRankSource pageRankSource = new PageRankSource();
    PageRankKeyedReduce pageRankKeyedReduce = new PageRankKeyedReduce();
    PagerankSink pagerankSink = new PagerankSink();
    ComputeGraphBuilder pagerankComputationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    pagerankComputationTaskGraphBuilder.addSource("pageranksource", pageRankSource, parallelismValue);
    ComputeConnection computeConnectionKeyedReduce = pagerankComputationTaskGraphBuilder.addCompute("pagerankcompute", pageRankKeyedReduce, parallelismValue);
    ComputeConnection computeConnectionAllReduce = pagerankComputationTaskGraphBuilder.addCompute("pageranksink", pagerankSink, parallelismValue);
    computeConnectionKeyedReduce.keyedReduce("pageranksource").viaEdge("keyedreduce").withReductionFunction(new ReduceFn(Op.SUM, MessageTypes.DOUBLE_ARRAY)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.DOUBLE_ARRAY);
    computeConnectionAllReduce.allreduce("pagerankcompute").viaEdge("all-reduce").withReductionFunction(new Aggregate()).withDataType(MessageTypes.OBJECT);
    pagerankComputationTaskGraphBuilder.setMode(OperationMode.BATCH);
    pagerankComputationTaskGraphBuilder.setTaskGraphName("buildComputationTG");
    return pagerankComputationTaskGraphBuilder.build();
}

Example 3 with ReduceFn

use of edu.iu.dsc.tws.task.impl.function.ReduceFn in project twister2 by DSC-SPIDAL.

the class WordCountJob method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    // source and aggregator
    WordSource source = new WordSource();
    WordAggregator counter = new WordAggregator();
    // build the task graph
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    builder.addSource("word-source", source, 4);
    builder.addCompute("word-aggregator", counter, 4).keyedReduce("word-source").viaEdge(EDGE).withReductionFunction(new ReduceFn(Op.SUM, MessageTypes.INTEGER_ARRAY)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.INTEGER_ARRAY);
    builder.setMode(OperationMode.BATCH);
    // execute the graph
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = cEnv.getTaskExecutor().plan(graph);
    cEnv.getTaskExecutor().execute(graph, plan);
}