Examples with ComputeGraphBuilder edu.iu.dsc.tws.task.impl.ComputeGraphBuilder used on opensource projects

Example 46 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class DataLocalityTaskSchedulerTest method createGraphWithComputeTaskAndConstraints.

private ComputeGraph createGraphWithComputeTaskAndConstraints(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestCompute testCompute = new TaskSchedulerClassTest.TestCompute();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    computeGraphBuilder.addSource("source", testSource, parallel);
    ComputeConnection computeConnection = computeGraphBuilder.addCompute("compute", testCompute, parallel);
    ComputeConnection sinkComputeConnection = computeGraphBuilder.addCompute("sink", testSink, parallel);
    computeConnection.direct("source").viaEdge("cdirect-edge").withDataType(MessageTypes.OBJECT);
    sinkComputeConnection.direct("compute").viaEdge("sdirect-edge").withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.STREAMING);
    computeGraphBuilder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "10");
    ComputeGraph taskGraph = computeGraphBuilder.build();
    return taskGraph;
}

Example 47 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class IterativeJob method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    IterativeSourceTask g = new IterativeSourceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder graphBuilder = ComputeGraphBuilder.newBuilder(config);
    graphBuilder.addSource("source", g, 4);
    ComputeConnection computeConnection = graphBuilder.addCompute("sink", r, 4);
    computeConnection.partition("source").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph graph = graphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    IExecutor ex = taskExecutor.createExecution(graph, plan);
    for (int i = 0; i < 10; i++) {
        LOG.info("Starting iteration: " + i);
        taskExecutor.addInput(graph, plan, "source", "input", new DataObjectImpl<>(config));
        // this is a blocking call
        ex.execute();
        DataObject<Object> dataSet = taskExecutor.getOutput(graph, plan, "sink");
        DataPartition<Object>[] values = dataSet.getPartitions();
    }
    ex.closeExecution();
}

Example 48 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class TGUtils method buildWeightVectorTG.

public static ComputeGraph buildWeightVectorTG(Config config, int dataStreamerParallelism, SVMJobParameters svmJobParameters, OperationMode opMode) {
    DataFileReplicatedReadSource dataFileReplicatedReadSource = new DataFileReplicatedReadSource(Context.TWISTER2_DIRECT_EDGE, svmJobParameters.getWeightVectorDataDir());
    IterativeSVMWeightVectorObjectCompute weightVectorObjectCompute = new IterativeSVMWeightVectorObjectCompute(Context.TWISTER2_DIRECT_EDGE, 1, svmJobParameters.getFeatures());
    IterativeSVMWeightVectorObjectDirectSink weightVectorObjectSink = new IterativeSVMWeightVectorObjectDirectSink();
    ComputeGraphBuilder weightVectorComputeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    weightVectorComputeGraphBuilder.addSource(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_SOURCE, dataFileReplicatedReadSource, dataStreamerParallelism);
    ComputeConnection weightVectorComputeConnection = weightVectorComputeGraphBuilder.addCompute(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_COMPUTE, weightVectorObjectCompute, dataStreamerParallelism);
    ComputeConnection weightVectorSinkConnection = weightVectorComputeGraphBuilder.addCompute(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_SINK, weightVectorObjectSink, dataStreamerParallelism);
    weightVectorComputeConnection.direct(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_SOURCE).viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    weightVectorSinkConnection.direct(Constants.SimpleGraphConfig.WEIGHT_VECTOR_OBJECT_COMPUTE).viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.DOUBLE_ARRAY);
    weightVectorComputeGraphBuilder.setMode(opMode);
    weightVectorComputeGraphBuilder.setTaskGraphName(IterativeSVMConstants.WEIGHT_VECTOR_LOADING_TASK_GRAPH);
    return weightVectorComputeGraphBuilder.build();
}

Example 49 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class SvmSgdRunner method initializeExecute.

/**
 * Initializing the execute method
 */
public void initializeExecute() {
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    this.operationMode = this.svmJobParameters.isStreaming() ? OperationMode.STREAMING : OperationMode.BATCH;
    DataStreamer dataStreamer = new DataStreamer(this.operationMode, svmJobParameters.isDummy(), this.binaryBatchModel);
    SVMCompute svmCompute = new SVMCompute(this.binaryBatchModel, this.operationMode);
    SVMReduce svmReduce = new SVMReduce(this.operationMode);
    builder.addSource(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, dataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_COMPUTE, svmCompute, svmComputeParallelism);
    ComputeConnection svmReduceConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_REDUCE, svmReduce, reduceParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.DATA_EDGE).withDataType(MessageTypes.OBJECT);
    svmReduceConnection.reduce(Constants.SimpleGraphConfig.SVM_COMPUTE).viaEdge(Constants.SimpleGraphConfig.REDUCE_EDGE).withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(operationMode);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    taskExecutor.execute(graph, plan);
    LOG.info("Task Graph Executed !!! ");
    if (operationMode.equals(OperationMode.BATCH)) {
        DataObject<double[]> dataSet = taskExecutor.getOutput(graph, plan, Constants.SimpleGraphConfig.SVM_REDUCE);
        DataPartition<double[]> values = dataSet.getPartitions()[0];
        DataPartitionConsumer<double[]> dataPartitionConsumer = values.getConsumer();
        // LOG.info("Final Receive  : " + dataPartitionConsumer.hasNext());
        while (dataPartitionConsumer.hasNext()) {
            LOG.info("Final Aggregated Values Are:" + Arrays.toString(dataPartitionConsumer.next()));
        }
    }
}

Example 50 with ComputeGraphBuilder

use of edu.iu.dsc.tws.task.impl.ComputeGraphBuilder in project twister2 by DSC-SPIDAL.

the class DataParallelWorker method execute.

@Override
public void execute() {
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    String inputDirectory = config.getStringValue(Constants.ARGS_INPUT_DIRECTORY);
    boolean shared = config.getBooleanValue(Constants.ARGS_SHARED_FILE_SYSTEM);
    int numFiles = config.getIntegerValue(Constants.ARGS_NUMBER_OF_FILES, 4);
    int size = config.getIntegerValue(Constants.ARGS_SIZE, 1000);
    int parallel = config.getIntegerValue(Constants.ARGS_PARALLEL, 2);
    if (!shared && workerId == 0) {
        try {
            DataGenerator.generateData("txt", new Path(inputDirectory), numFiles, size, 10);
        } catch (IOException e) {
            throw new RuntimeException("Failed to create data: " + inputDirectory);
        }
    }
    DataParallelTask task = new DataParallelTask();
    computeGraphBuilder.addSource("map", task, parallel);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph computeGraph = computeGraphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(computeGraph);
    taskExecutor.execute(computeGraph, plan);
}