Examples with ExecutionPlan edu.iu.dsc.tws.api.compute.executor.ExecutionPlan used on opensource projects

Example 6 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdIterativeRunner method loadTestingData.

private void loadTestingData() {
    ComputeGraph testingDatapointsTaskGraph = buildTestingDataPointsTG();
    ExecutionPlan datapointsExecutionPlan = taskExecutor.plan(testingDatapointsTaskGraph);
    taskExecutor.execute(testingDatapointsTaskGraph, datapointsExecutionPlan);
    testingDoubleDataPointObject = taskExecutor.getOutput(testingDatapointsTaskGraph, datapointsExecutionPlan, Constants.SimpleGraphConfig.DATA_OBJECT_SINK_TESTING);
    double[][] datapoints = null;
    for (int i = 0; i < testingDoubleDataPointObject.getPartitions().length; i++) {
        datapoints = testingDoubleDataPointObject.getPartitions()[i].getConsumer().next();
        LOG.info(String.format("Partition[%d] Testing Datapoints : %d,%d", i, datapoints.length, datapoints[0].length));
    }
    System.out.println("---------Testing Data-------------");
    double sum = 0;
    int count = 0;
    for (int j = 0; j < datapoints.length; j++) {
        sum = Matrix.sum(datapoints[j]);
        if (sum == 0) {
            count++;
        }
    }
    System.out.println(String.format("%d,%d, Zero Sum: %d", datapoints.length, datapoints[0].length, count));
}

Example 7 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdIterativeRunner method loadTrainingData.

private void loadTrainingData() {
    ComputeGraph trainingDatapointsTaskGraph = buildTrainingDataPointsTG();
    ExecutionPlan datapointsExecutionPlan = taskExecutor.plan(trainingDatapointsTaskGraph);
    taskExecutor.execute(trainingDatapointsTaskGraph, datapointsExecutionPlan);
    trainingDoubleDataPointObject = taskExecutor.getOutput(trainingDatapointsTaskGraph, datapointsExecutionPlan, Constants.SimpleGraphConfig.DATA_OBJECT_SINK);
    double[][] datapoints = null;
    for (int i = 0; i < trainingDoubleDataPointObject.getPartitions().length; i++) {
        datapoints = trainingDoubleDataPointObject.getPartitions()[i].getConsumer().next();
        LOG.info(String.format("Training Datapoints : %d,%d", datapoints.length, datapoints[0].length));
    }
    System.out.println("---------Training Data-------------");
    double sum = 0;
    int count = 0;
    for (int j = 0; j < datapoints.length; j++) {
        sum = Matrix.sum(datapoints[j]);
        if (sum == 0) {
            count++;
        }
    }
    System.out.println(String.format("Training: %d,%d, Zero Sum: %d", datapoints.length, datapoints[0].length, count));
}

Example 8 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SourceTaskDataLoader method execute.

@Override
public void execute() {
    getParams();
    /*
     * First data is loaded from files
     * */
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    // DataObjectSource sourceTask = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE,
    // dataSource);
    // DataObjectSink sinkTask = new DataObjectSink();
    // computeGraphBuilder.addSource("datapointsource", sourceTask, parallelism);
    // ComputeConnection firstGraphComputeConnection = computeGraphBuilder.addSink(
    // "datapointsink", sinkTask, parallelism);
    // firstGraphComputeConnection.direct("datapointsource",
    // Context.TWISTER2_DIRECT_EDGE, DataType.OBJECT);
    // computeGraphBuilder.setMode(OperationMode.BATCH);
    // 
    // ComputeGraph datapointsTaskGraph = computeGraphBuilder.build();
    // ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
    // taskExecutor.execute(datapointsTaskGraph, firstGraphExecutionPlan);
    // DataObject<Object> dataPointsObject = taskExecutor.getOutput(
    // datapointsTaskGraph, firstGraphExecutionPlan, "datapointsink");
    // LOG.info("Total Partitions : " + dataPointsObject.getPartitions().length);
    /*
     * Second Task
     * */
    DataSourceTask kMeansSourceTask = new DataSourceTask();
    SimpleDataAllReduceTask kMeansAllReduceTask = new SimpleDataAllReduceTask();
    computeGraphBuilder.addSource("kmeanssource", kMeansSourceTask, parallelism);
    ComputeConnection computeConnection = computeGraphBuilder.addCompute("kmeanssink", kMeansAllReduceTask, parallelism);
    computeConnection.allreduce("kmeanssource").viaEdge("all-reduce").withReductionFunction(new SimpleDataAggregator()).withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph simpleTaskGraph = computeGraphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(simpleTaskGraph);
    // taskExecutor.addInput(
    // simpleTaskGraph, plan, "kmeanssource", "points", dataPointsObject);
    taskExecutor.execute(simpleTaskGraph, plan);
    DataObject<double[][]> dataSet = taskExecutor.getOutput(simpleTaskGraph, plan, "kmeanssink");
// DataObject<Object> dataSet = taskExecutor.getOutput(simpleTaskGraph, plan, "kmeanssink");
// DataPartition<Object> values = dataSet.getPartitions()[0];
// Object lastObject = values.getConsumer().next();
// LOG.info(String.format("Last Object : %s", lastObject.getClass().getGraphName()));
}

Example 9 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class TaskWorkerDataLoader method execute.

@Override
public void execute() {
    getParams();
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    DataObjectSource sourceTask = new DataObjectSource(Context.TWISTER2_DIRECT_EDGE, dataSource);
    DataObjectSink sinkTask = new DataObjectSink();
    computeGraphBuilder.addSource("datapointsource", sourceTask, parallelism);
    ComputeConnection firstGraphComputeConnection = computeGraphBuilder.addCompute("datapointsink", sinkTask, parallelism);
    firstGraphComputeConnection.direct("datapointsource").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph datapointsTaskGraph = computeGraphBuilder.build();
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
    taskExecutor.execute(datapointsTaskGraph, firstGraphExecutionPlan);
    DataObject<Object> dataPointsObject = taskExecutor.getOutput(datapointsTaskGraph, firstGraphExecutionPlan, "datapointsink");
    LOG.info("Total Partitions : " + dataPointsObject.getPartitions().length);
    showAllUnits(dataPointsObject);
}

Example 10 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdAdvancedRunner method executeTestingDataLoadingTaskGraph.

/**
 * This method loads the testing data
 * The loaded test data is used to evaluate the trained data
 * Testing data is loaded in parallel depending on the parallelism parameter given
 * There are partitions created equal to the parallelism
 * Later this will be used to do the testing in parallel in the testing task graph
 *
 * @return twister2 DataObject containing the testing data
 */
public DataObject<Object> executeTestingDataLoadingTaskGraph() {
    DataObject<Object> data = null;
    final String TEST_DATA_LOAD_EDGE_DIRECT = "direct2";
    DataObjectSource sourceTask1 = new DataObjectSource(TEST_DATA_LOAD_EDGE_DIRECT, this.svmJobParameters.getTestingDataDir());
    DataObjectSink sinkTask1 = new DataObjectSink();
    testingBuilder.addSource(Constants.SimpleGraphConfig.DATA_OBJECT_SOURCE_TESTING, sourceTask1, dataStreamerParallelism);
    ComputeConnection firstGraphComputeConnection1 = testingBuilder.addCompute(Constants.SimpleGraphConfig.DATA_OBJECT_SINK_TESTING, sinkTask1, dataStreamerParallelism);
    firstGraphComputeConnection1.direct(Constants.SimpleGraphConfig.DATA_OBJECT_SOURCE_TESTING).viaEdge(TEST_DATA_LOAD_EDGE_DIRECT).withDataType(MessageTypes.OBJECT);
    testingBuilder.setMode(OperationMode.BATCH);
    ComputeGraph datapointsTaskGraph1 = testingBuilder.build();
    datapointsTaskGraph1.setGraphName("testing-data-loading-graph");
    ExecutionPlan firstGraphExecutionPlan1 = taskExecutor.plan(datapointsTaskGraph1);
    taskExecutor.execute(datapointsTaskGraph1, firstGraphExecutionPlan1);
    data = taskExecutor.getOutput(datapointsTaskGraph1, firstGraphExecutionPlan1, Constants.SimpleGraphConfig.DATA_OBJECT_SINK_TESTING);
    if (data == null) {
        throw new NullPointerException("Something Went Wrong in Loading Testing Data");
    } else {
        LOG.info("Testing Data Total Partitions : " + data.getPartitions().length);
    }
    return data;
}