Examples with ComputeGraphBuilder edu.iu.dsc.tws.task.impl.ComputeGraphBuilder used on opensource projects

Example 31 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class KMeansConnectedDataflowExample method generateFirstJob.

private static DataFlowGraph generateFirstJob(Config config, int parallelismValue, String dataDirectory, int dimension, int dsize, int instances, DataFlowJobConfig jobConfig) {
    DataObjectSource dataObjectSource = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE, dataDirectory);
    KMeansDataObjectCompute dataObjectCompute = new KMeansDataObjectCompute(Context.TWISTER2_DIRECT_EDGE, dsize, parallelismValue, dimension);
    KMeansDataObjectDirectSink dataObjectSink = new KMeansDataObjectDirectSink("points");
    ComputeGraphBuilder datapointsComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    // Add source, compute, and sink tasks to the task graph builder for the first task graph
    datapointsComputeGraphBuilder.addSource("datapointsource", dataObjectSource, parallelismValue);
    ComputeConnection datapointComputeConnection = datapointsComputeGraphBuilder.addCompute("datapointcompute", dataObjectCompute, parallelismValue);
    ComputeConnection firstGraphComputeConnection = datapointsComputeGraphBuilder.addCompute("datapointsink", dataObjectSink, parallelismValue);
    // Creating the communication edges between the tasks for the second task graph
    datapointComputeConnection.direct("datapointsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    firstGraphComputeConnection.direct("datapointcompute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointsComputeGraphBuilder.setMode(OperationMode.BATCH);
    datapointsComputeGraphBuilder.setTaskGraphName("datapointsTG");
    ComputeGraph firstGraph = datapointsComputeGraphBuilder.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("datapointsTG", firstGraph).setWorkers(instances).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 32 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class TaskGraphBuildTest method createGraphWithEdgeName.

private ComputeGraph createGraphWithEdgeName(String edgeName) {
    TestSource testSource = new TestSource();
    TestSink1 testCompute = new TestSink1();
    TestSink2 testSink = new TestSink2();
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(getConfig());
    computeGraphBuilder.addSource("source", testSource, 4);
    ComputeConnection computeConnection = computeGraphBuilder.addCompute("compute", testCompute, 4);
    computeConnection.partition("source").viaEdge(edgeName).withDataType(MessageTypes.OBJECT);
    ComputeConnection rc = computeGraphBuilder.addCompute("sink", testSink, 1);
    rc.allreduce("compute").viaEdge(edgeName).withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    ComputeGraph graph = computeGraphBuilder.build();
    return graph;
}

Example 33 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class MultiStageGraph method execute.

@Override
public void execute() {
    GeneratorTask g = new GeneratorTask();
    ReduceTask rt = new ReduceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    builder.addSource("source", g, 4);
    ComputeConnection pc = builder.addCompute("compute", r, 4);
    pc.partition("source").viaEdge("partition-edge").withDataType(MessageTypes.OBJECT);
    ComputeConnection rc = builder.addCompute("sink", rt, 1);
    rc.reduce("compute").viaEdge("compute-edge").withReductionFunction((object1, object2) -> object1);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    graph.setGraphName("MultiTaskGraph");
    ExecutionPlan plan = taskExecutor.plan(graph);
    taskExecutor.execute(graph, plan);
}

Example 34 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class TGUtils method generateGenericDataPointLoader.

public static ComputeGraph generateGenericDataPointLoader(int samples, int parallelism, int numOfFeatures, String dataSourcePathStr, String dataObjectSourceStr, String dataObjectComputeStr, String dataObjectSinkStr, String graphName, Config config, OperationMode opMode) {
    SVMDataObjectSource<String, TextInputSplit> sourceTask = new SVMDataObjectSource(Context.TWISTER2_DIRECT_EDGE, dataSourcePathStr, samples);
    IterativeSVMDataObjectCompute dataObjectCompute = new IterativeSVMDataObjectCompute(Context.TWISTER2_DIRECT_EDGE, parallelism, samples, numOfFeatures, DELIMITER);
    IterativeSVMDataObjectDirectSink iterativeSVMPrimaryDataObjectDirectSink = new IterativeSVMDataObjectDirectSink();
    ComputeGraphBuilder datapointsComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    datapointsComputeGraphBuilder.addSource(dataObjectSourceStr, sourceTask, parallelism);
    ComputeConnection datapointComputeConnection = datapointsComputeGraphBuilder.addCompute(dataObjectComputeStr, dataObjectCompute, parallelism);
    ComputeConnection computeConnectionSink = datapointsComputeGraphBuilder.addCompute(dataObjectSinkStr, iterativeSVMPrimaryDataObjectDirectSink, parallelism);
    datapointComputeConnection.direct(dataObjectSourceStr).viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    computeConnectionSink.direct(dataObjectComputeStr).viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    datapointsComputeGraphBuilder.setMode(opMode);
    datapointsComputeGraphBuilder.setTaskGraphName(graphName);
    // Build the first taskgraph
    return datapointsComputeGraphBuilder.build();
}

Example 35 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class SvmSgdIterativeRunner method buildWeightVectorTG.

private ComputeGraph buildWeightVectorTG() {
    DataFileReplicatedReadSource dataFileReplicatedReadSource = new DataFileReplicatedReadSource(Context.TWISTER2_DIRECT_EDGE, this.svmJobParameters.getWeightVectorDataDir(), 1);
    IterativeSVMWeightVectorObjectCompute weightVectorObjectCompute = new IterativeSVMWeightVectorObjectCompute(Context.TWISTER2_DIRECT_EDGE, 1, this.svmJobParameters.getFeatures());
    IterativeSVMWeightVectorObjectDirectSink weightVectorObjectSink = new IterativeSVMWeightVectorObjectDirectSink();
    ComputeGraphBuilder weightVectorComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    weightVectorComputeGraphBuilder.addSource(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_SOURCE, dataFileReplicatedReadSource, dataStreamerParallelism);
    ComputeConnection weightVectorComputeConnection = weightVectorComputeGraphBuilder.addCompute(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_COMPUTE, weightVectorObjectCompute, dataStreamerParallelism);
    ComputeConnection weightVectorSinkConnection = weightVectorComputeGraphBuilder.addCompute(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_SINK, weightVectorObjectSink, dataStreamerParallelism);
    weightVectorComputeConnection.direct(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_SOURCE).viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    weightVectorSinkConnection.direct(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_COMPUTE).viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.DOUBLE_ARRAY);
    weightVectorComputeGraphBuilder.setMode(operationMode);
    weightVectorComputeGraphBuilder.setTaskGraphName(IterativeSVMConstants.WEIGHT_VECTOR_LOADING_TASK_GRAPH);
    return weightVectorComputeGraphBuilder.build();
}