Examples with ComputeConnection edu.iu.dsc.tws.task.impl.ComputeConnection used on opensource projects

Example 56 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class KMeansConnectedDataflowExample method generateSecondJob.

private static DataFlowGraph generateSecondJob(Config config, int parallelismValue, String centroidDirectory, int dimension, int csize, int instances, DataFlowJobConfig jobConfig) {
    DataFileReplicatedReadSource dataFileReplicatedReadSource = new DataFileReplicatedReadSource(Context.TWISTER2_DIRECT_EDGE, centroidDirectory);
    KMeansDataObjectCompute centroidObjectCompute = new KMeansDataObjectCompute(Context.TWISTER2_DIRECT_EDGE, csize, dimension);
    KMeansDataObjectDirectSink centroidObjectSink = new KMeansDataObjectDirectSink("centroids");
    ComputeGraphBuilder centroidsComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    // Add source, compute, and sink tasks to the task graph builder for the second task graph
    centroidsComputeGraphBuilder.addSource("centroidsource", dataFileReplicatedReadSource, parallelismValue);
    ComputeConnection centroidComputeConnection = centroidsComputeGraphBuilder.addCompute("centroidcompute", centroidObjectCompute, parallelismValue);
    ComputeConnection secondGraphComputeConnection = centroidsComputeGraphBuilder.addCompute("centroidsink", centroidObjectSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    centroidComputeConnection.direct("centroidsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    secondGraphComputeConnection.direct("centroidcompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    centroidsComputeGraphBuilder.setMode(OperationMode.BATCH);
    centroidsComputeGraphBuilder.setTaskGraphName("centroidTG");
    // Build the second taskgraph
    ComputeGraph secondGraph = centroidsComputeGraphBuilder.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("centroidTG", secondGraph).setWorkers(instances).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 57 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class TopologyBuilder method createTopology.

public StormTopology createTopology() {
    this.sourceNodes.forEach(source -> {
        Twister2Spout twister2Spout = (Twister2Spout) nodes.get(source);
        LOG.info("Adding source : " + source);
        this.computeGraphBuilder.addSource(source, twister2Spout, twister2Spout.getParallelism());
    });
    this.computeNodes.forEach(compute -> {
        Twister2Bolt twister2Bolt = (Twister2Bolt) nodes.get(compute);
        ComputeConnection computeConnection = this.computeGraphBuilder.addCompute(compute, twister2Bolt, twister2Bolt.getParallelism());
        this.defineGrouping(twister2Bolt, computeConnection);
    });
    this.sinkNodes.forEach(sink -> {
        Twister2Bolt twister2Bolt = (Twister2Bolt) nodes.get(sink);
        ComputeConnection computeConnection = this.computeGraphBuilder.addCompute(sink, twister2Bolt, twister2Bolt.getParallelism());
        this.defineGrouping(twister2Bolt, computeConnection);
    });
    this.computeGraphBuilder.setMode(OperationMode.STREAMING);
    return new StormTopology(this.computeGraphBuilder.build());
}

Example 58 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class PageRankWorker method buildComputationTG.

public static ComputeGraph buildComputationTG(int parallelismValue, Config conf) {
    PageRankSource pageRankSource = new PageRankSource();
    PageRankKeyedReduce pageRankKeyedReduce = new PageRankKeyedReduce();
    PagerankSink pagerankSink = new PagerankSink();
    ComputeGraphBuilder pagerankComputationTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    pagerankComputationTaskGraphBuilder.addSource("pageranksource", pageRankSource, parallelismValue);
    ComputeConnection computeConnectionKeyedReduce = pagerankComputationTaskGraphBuilder.addCompute("pagerankcompute", pageRankKeyedReduce, parallelismValue);
    ComputeConnection computeConnectionAllReduce = pagerankComputationTaskGraphBuilder.addCompute("pageranksink", pagerankSink, parallelismValue);
    computeConnectionKeyedReduce.keyedReduce("pageranksource").viaEdge("keyedreduce").withReductionFunction(new ReduceFn(Op.SUM, MessageTypes.DOUBLE_ARRAY)).withKeyType(MessageTypes.OBJECT).withDataType(MessageTypes.DOUBLE_ARRAY);
    computeConnectionAllReduce.allreduce("pagerankcompute").viaEdge("all-reduce").withReductionFunction(new Aggregate()).withDataType(MessageTypes.OBJECT);
    pagerankComputationTaskGraphBuilder.setMode(OperationMode.BATCH);
    pagerankComputationTaskGraphBuilder.setTaskGraphName("buildComputationTG");
    return pagerankComputationTaskGraphBuilder.build();
}

Example 59 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class SingleSourceShortestPathWorker method buildDataPointsTG.

public static ComputeGraph buildDataPointsTG(String dataDirectory, int dsize, int parallelismValue, String soruceVertex, Config conf) {
    GraphDataSource dataObjectSource = new GraphDataSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory, dsize);
    GraphDataCompute graphDataCompute = new GraphDataCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue, soruceVertex);
    GraphDataSink graphDataSink = new GraphDataSink("PartitionSink");
    ComputeGraphBuilder datapointsTaskGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    datapointsTaskGraphBuilder.addSource("Graphdatasource", dataObjectSource, parallelismValue);
    ComputeConnection datapointComputeConnection = datapointsTaskGraphBuilder.addCompute("Graphdatacompute", graphDataCompute, parallelismValue);
    ComputeConnection firstGraphComputeConnection = datapointsTaskGraphBuilder.addCompute("Graphdatasink", graphDataSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("Graphdatasource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("Graphdatacompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointsTaskGraphBuilder.setMode(OperationMode.BATCH);
    datapointsTaskGraphBuilder.setTaskGraphName("datapointsTG");
    // Build the first taskgraph
    return datapointsTaskGraphBuilder.build();
}

Example 60 with ComputeConnection

use of edu.iu.dsc.tws.task.impl.ComputeConnection in project twister2 by DSC-SPIDAL.

the class ConstraintTaskExample method buildFirstGraph.

private ComputeGraph buildFirstGraph(int parallelism, Config conf, String dataInput, int dataSize, int dimension, String inputKey, String constraint) {
    FirstSourceTask sourceTask = new FirstSourceTask(dataInput, dataSize);
    FirstSinkTask sinkTask = new FirstSinkTask(dimension, inputKey);
    ComputeGraphBuilder firstGraphBuilder = ComputeGraphBuilder.newBuilder(conf);
    firstGraphBuilder.addSource("firstsource", sourceTask, parallelism);
    ComputeConnection computeConnection = firstGraphBuilder.addCompute("firstsink", sinkTask, parallelism);
    computeConnection.direct("firstsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphBuilder.setMode(OperationMode.BATCH);
    firstGraphBuilder.setTaskGraphName("firstTG");
    firstGraphBuilder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, constraint);
    return firstGraphBuilder.build();
}