Examples with ExecutionPlan edu.iu.dsc.tws.api.compute.executor.ExecutionPlan used on opensource projects

Example 31 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class DataParallelWorker method execute.

@Override
public void execute() {
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    String inputDirectory = config.getStringValue(Constants.ARGS_INPUT_DIRECTORY);
    boolean shared = config.getBooleanValue(Constants.ARGS_SHARED_FILE_SYSTEM);
    int numFiles = config.getIntegerValue(Constants.ARGS_NUMBER_OF_FILES, 4);
    int size = config.getIntegerValue(Constants.ARGS_SIZE, 1000);
    int parallel = config.getIntegerValue(Constants.ARGS_PARALLEL, 2);
    if (!shared && workerId == 0) {
        try {
            DataGenerator.generateData("txt", new Path(inputDirectory), numFiles, size, 10);
        } catch (IOException e) {
            throw new RuntimeException("Failed to create data: " + inputDirectory);
        }
    }
    DataParallelTask task = new DataParallelTask();
    computeGraphBuilder.addSource("map", task, parallel);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph computeGraph = computeGraphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(computeGraph);
    taskExecutor.execute(computeGraph, plan);
}

Example 32 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SingleSourceShortestPathWorker method execute.

@Override
public void execute() {
    SsspParameters ssspParameters = SsspParameters.build(config);
    int parallelismValue = ssspParameters.getParallelismValue();
    int dsize = ssspParameters.getDsize();
    String dataDirectory = ssspParameters.getDatapointDirectory();
    String soruceVertex = ssspParameters.getSourcevertex();
    sourceVertexGlobal = soruceVertex;
    /* First Graph to partition and read the partitioned adjacency list datas **/
    // Build the first taskgraph
    ComputeGraph datapointsTaskGraph = buildDataPointsTG(dataDirectory, dsize, parallelismValue, soruceVertex, config);
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(datapointsTaskGraph, firstGraphExecutionPlan);
    // Retrieve the output of the first task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // Build the second taskgraph
    ComputeGraph graphInitialValueTaskGraph = buildSsspInitialTG(dataDirectory, dsize, parallelismValue, soruceVertex, config);
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(graphInitialValueTaskGraph);
    // Actual execution for the second taskgraph
    taskExecutor.execute(graphInitialValueTaskGraph, secondGraphExecutionPlan);
    // Retrieve the output of the second task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    /* Third Graph to do the actual calculation **/
    ComputeGraph sssptaskgraph = buildComputationSsspTG(parallelismValue, config);
    IExecutor ex = taskExecutor.createExecution(sssptaskgraph);
    int itr = 0;
    while (globaliterationStatus) {
        ex.execute(false);
        itr++;
    }
    ex.close();
    taskExecutor.close();
    if (workerId == 1) {
        System.out.println("Tatol iteration: " + itr);
    }
}

Example 33 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class ConstraintTaskExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    long startTime = System.currentTimeMillis();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    String dinput = String.valueOf(config.get(DataObjectConstants.DINPUT_DIRECTORY));
    int dimension = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.DIMENSIONS)));
    int parallelismValue = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.PARALLELISM_VALUE)));
    int dsize = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.DSIZE)));
    DataGenerator dataGenerator = new DataGenerator(config, workerId);
    dataGenerator.generate(new Path(dinput), dsize, dimension);
    ComputeGraph firstGraph = buildFirstGraph(parallelismValue, config, dinput, dsize, dimension, "firstgraphpoints", "1");
    ComputeGraph secondGraph = buildSecondGraph(parallelismValue, config, dimension, "firstgraphpoints", "1");
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(firstGraph);
    taskExecutor.execute(firstGraph, firstGraphExecutionPlan);
    DataObject<Object> firstGraphObject = taskExecutor.getOutput("firstsink");
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(secondGraph);
    taskExecutor.addInput("firstgraphpoints", firstGraphObject);
    taskExecutor.execute(secondGraph, secondGraphExecutionPlan);
    long endTime = System.currentTimeMillis();
    LOG.info("Total Execution Time: " + (endTime - startTime));
}

Example 34 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class DataflowNodeExperiment method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    SourceTask sourceTask = new SourceTask();
    ReduceTask reduceTask = new ReduceTask();
    ComputeTask computeTask = new ComputeTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    DataflowJobParameters dataflowJobParameters = DataflowJobParameters.build(config);
    int parallel = dataflowJobParameters.getParallelismValue();
    int iter = dataflowJobParameters.getIterations();
    builder.addSource("source", sourceTask, parallel);
    ComputeConnection computeConnection = builder.addCompute("compute", computeTask, parallel);
    ComputeConnection rc = builder.addCompute("sink", reduceTask, parallel);
    computeConnection.direct("source").viaEdge("direct").withDataType(MessageTypes.OBJECT);
    rc.allreduce("compute").viaEdge("all-reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < iter; i++) {
        taskExecutor.execute(graph, plan);
        LOG.info("Completed Iteration:" + i);
    }
    long stopTime = System.currentTimeMillis();
    long executionTime = stopTime - startTime;
    LOG.info("Total Execution Time to Complete Dataflow Node Experiment" + "\t" + executionTime + "(in milliseconds)");
}

Example 35 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class KMeansCheckpointingWorker method execute.

/**
 * First, the execute method invokes the generateDataPoints method to generate the datapoints file
 * and centroid file based on the respective filesystem submitted by the user. Next, it invoke
 * the DataObjectSource and DataObjectSink to partition and read the partitioned data points
 * respectively through data points task graph. Then, it calls the DataFileReader to read the
 * centroid values from the filesystem through centroid task graph. Next, the datapoints are
 * stored in DataSet \(0th object\) and centroids are stored in DataSet 1st object\). Finally, it
 * constructs the kmeans task graph to perform the clustering process which computes the distance
 * between the centroids and data points.
 */
@SuppressWarnings("unchecked")
@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    IWorkerController workerController = workerEnv.getWorkerController();
    ComputeEnvironment taskEnv = ComputeEnvironment.init(workerEnv);
    CheckpointingWorkerEnv checkpointingEnv = CheckpointingWorkerEnv.newBuilder(config, workerId, workerController).registerVariable(I_KEY, IntegerPacker.getInstance()).registerVariable(CENT_OBJ, ObjectPacker.getInstance()).build();
    Snapshot snapshot = checkpointingEnv.getSnapshot();
    TaskExecutor taskExecutor = taskEnv.getTaskExecutor();
    LOG.info("Task worker starting: " + workerId + " Current snapshot ver: " + snapshot.getVersion());
    int parallelismValue = config.getIntegerValue(DataObjectConstants.PARALLELISM_VALUE);
    int dimension = config.getIntegerValue(DataObjectConstants.DIMENSIONS);
    int numFiles = config.getIntegerValue(DataObjectConstants.NUMBER_OF_FILES);
    int dsize = config.getIntegerValue(DataObjectConstants.DSIZE);
    int csize = config.getIntegerValue(DataObjectConstants.CSIZE);
    int iterations = config.getIntegerValue(DataObjectConstants.ARGS_ITERATIONS);
    String dataDirectory = config.getStringValue(DataObjectConstants.DINPUT_DIRECTORY) + workerId;
    String centroidDirectory = config.getStringValue(DataObjectConstants.CINPUT_DIRECTORY) + workerId;
    String type = config.getStringValue(DataObjectConstants.FILE_TYPE);
    KMeansUtils.generateDataPoints(config, dimension, numFiles, dsize, csize, dataDirectory, centroidDirectory, type);
    long startTime = System.currentTimeMillis();
    /* First Graph to partition and read the partitioned data points **/
    ComputeGraph datapointsTaskGraph = KMeansComputeJob.buildDataPointsTG(dataDirectory, dsize, parallelismValue, dimension, config, type);
    // Get the execution plan for the first task graph
    ExecutionPlan datapointsExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(datapointsTaskGraph, datapointsExecutionPlan);
    // Retrieve the output of the first task graph
    DataObject<Object> dataPointsObject = taskExecutor.getOutput(datapointsTaskGraph, datapointsExecutionPlan, "datapointsink");
    DataObject<Object> centroidsDataObject;
    if (!snapshot.checkpointAvailable(CENT_OBJ)) {
        /* Second Graph to read the centroids **/
        ComputeGraph centroidsTaskGraph = KMeansComputeJob.buildCentroidsTG(centroidDirectory, csize, parallelismValue, dimension, config, type);
        // Get the execution plan for the second task graph
        ExecutionPlan centroidsExecutionPlan = taskExecutor.plan(centroidsTaskGraph);
        // Actual execution for the second taskgraph
        taskExecutor.execute(centroidsTaskGraph, centroidsExecutionPlan);
        // Retrieve the output of the first task graph
        centroidsDataObject = taskExecutor.getOutput(centroidsTaskGraph, centroidsExecutionPlan, "centroidsink");
    } else {
        centroidsDataObject = (DataObject<Object>) snapshot.get(CENT_OBJ);
    }
    long endTimeData = System.currentTimeMillis();
    /* Third Graph to do the actual calculation **/
    ComputeGraph kmeansTaskGraph = KMeansComputeJob.buildKMeansTG(parallelismValue, config);
    // Perform the iterations from 0 to 'n' number of iterations
    IExecutor ex = taskExecutor.createExecution(kmeansTaskGraph);
    for (int i = 0; i < iterations; i++) {
        // actual execution of the third task graph
        ex.execute(i == iterations - 1);
    }
    DataPartition<?> centroidPartition = centroidsDataObject.getPartition(workerId);
    double[][] centroid = (double[][]) centroidPartition.getConsumer().next();
    long endTime = System.currentTimeMillis();
    if (workerId == 0) {
        LOG.info("Data Load time : " + (endTimeData - startTime) + "\n" + "Total Time : " + (endTime - startTime) + "Compute Time : " + (endTime - endTimeData));
    }
    LOG.info("Final Centroids After\t" + iterations + "\titerations\t" + Arrays.deepToString(centroid));
    taskEnv.close();
}