Examples with IExecutor edu.iu.dsc.tws.api.compute.executor.IExecutor used on opensource projects

Example 1 with IExecutor

use of edu.iu.dsc.tws.api.compute.executor.IExecutor in project twister2 by DSC-SPIDAL.

the class BasicComputation method execute.

@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    WorkerParameter workerParameter = WorkerParameter.build(config);
    parallelism = workerParameter.getParallelismValue();
    graphsize = workerParameter.getDsize();
    sourceVertexGlobal = workerParameter.getSourcevertex();
    dataDirectory = workerParameter.getDatapointDirectory();
    iterations = workerParameter.getIterations();
    /* First Graph to partition and read the partitioned data points **/
    ComputeGraph graphpartitionTaskGraph = graphpartition();
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan = taskExecutor.plan(graphpartitionTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(graphpartitionTaskGraph, executionPlan);
    // Retrieve the output of the first task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // the second task graph for assign initial pagerank values for vertex.
    ComputeGraph graphInitializationTaskGraph = graphInitialization();
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan1 = taskExecutor.plan(graphInitializationTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(graphInitializationTaskGraph, executionPlan1);
    // Retrieve the output of the first task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // third task graph for computations
    ComputeGraph computationTaskgraph = computation();
    IExecutor ex = taskExecutor.createExecution(computationTaskgraph);
    int unlimitedItr = 0;
    long startime = System.currentTimeMillis();
    if (iterations != 0) {
        for (int i = 0; i < iterations; i++) {
            ex.execute(i == iterations - 1);
        }
    } else {
        while (globaliterationStatus) {
            ex.execute(false);
            unlimitedItr++;
        }
        ex.closeExecution();
    }
    taskExecutor.close();
    long endTime = System.currentTimeMillis();
    if (workerId == 0) {
        System.out.println("total number of iterations : " + unlimitedItr);
        System.out.println("computation time: " + (endTime - startime));
    }
}

Example 2 with IExecutor

use of edu.iu.dsc.tws.api.compute.executor.IExecutor in project twister2 by DSC-SPIDAL.

the class PageRankWorker method execute.

// private static double danglingNodeValues;
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    PageRankWorkerParameters pageRankWorkerParameters = PageRankWorkerParameters.build(config);
    int parallelismValue = pageRankWorkerParameters.getParallelismValue();
    int dsize = pageRankWorkerParameters.getDsize();
    String dataDirectory = pageRankWorkerParameters.getDatapointDirectory();
    int iterations = pageRankWorkerParameters.getIterations();
    graphsize = dsize;
    /* First Graph to partition and read the partitioned data points **/
    ComputeGraph datapointsTaskGraph = buildDataPointsTG(dataDirectory, dsize, parallelismValue, config);
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan = taskExecutor.plan(datapointsTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(datapointsTaskGraph, executionPlan);
    /* the out of the first graph would like below
    * task Id: 0
    {1=[3, 4], 2=[3, 4, 5]}*/
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // the second task graph for assign initial pagerank values for vertex.
    ComputeGraph graphInitialValueTaskGraph = buildGraphInitialValueTG(dataDirectory, dsize, parallelismValue, config);
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan1 = taskExecutor.plan(graphInitialValueTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(graphInitialValueTaskGraph, executionPlan1);
    /* the output of second graph should like below
      initiate the pagerank value
    * {1=0.25, 2=0.25}
     */
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // third task graph for computations
    ComputeGraph pageranktaskgraph = buildComputationTG(parallelismValue, config);
    IExecutor ex = taskExecutor.createExecution(pageranktaskgraph);
    // Perform the iterations from 0 to 'n' number of iterations
    long startime = System.currentTimeMillis();
    for (int i = 0; i < iterations; i++) {
        ex.execute(i == iterations - 1);
    }
    taskExecutor.close();
    long endTime = System.currentTimeMillis();
    if (workerId == 0) {
        DataObject<Object> graphInitialPagerankValue = taskExecutor.getOutput("InitialValue");
        DataPartition<?> finaloutput = graphInitialPagerankValue.getPartition(workerId);
        HashMap<String, Double> finalone = (HashMap<String, Double>) finaloutput.getConsumer().next();
        System.out.println(finalone);
        LOG.info("Final output After " + iterations + "iterations ");
        Iterator it = finalone.entrySet().iterator();
        Double recivedFinalDanglingValue = finalone.get("danglingvalues");
        double cummulativepagerankvalue = 0.0;
        int num = 0;
        System.out.println(graphsize);
        while (it.hasNext()) {
            Map.Entry pair = (Map.Entry) it.next();
            if (!pair.getKey().equals("danglingvalues")) {
                double finalPagerankValue = (double) pair.getValue() + ((0.85 * recivedFinalDanglingValue) / graphsize);
                System.out.print("Vertex Id: " + pair.getKey());
                System.out.printf(" and it's pagerank value: %.15f \n", finalPagerankValue);
                cummulativepagerankvalue += finalPagerankValue;
                num += 1;
            }
            // avoids a ConcurrentModificationException
            it.remove();
        }
        System.out.println(recivedFinalDanglingValue);
        System.out.println(num);
        System.out.println(cummulativepagerankvalue);
        System.out.println(cummulativepagerankvalue + ((graphsize - num) * ((((double) 1 / graphsize) * 0.15) + (0.85 * (recivedFinalDanglingValue / graphsize)))));
        System.out.println("computation time: " + (endTime - startime));
    }
}

Example 3 with IExecutor

use of edu.iu.dsc.tws.api.compute.executor.IExecutor in project twister2 by DSC-SPIDAL.

the class TaskExecutor method execute.

/**
 * Execute a plan and a graph. This call blocks until the execution finishes. In case of
 * streaming, this call doesn't return while for batch computations it returns after
 * the execution is done.
 *
 * @param taskConfig the user configuration to be passed to the task instances
 * @param graph the dataflow graph
 * @param plan the execution plan
 */
public void execute(Config taskConfig, ComputeGraph graph, ExecutionPlan plan) {
    Config newCfg = Config.newBuilder().putAll(config).putAll(taskConfig).build();
    IExecutor ex = executor.getExecutor(newCfg, plan, graph.getOperationMode(), new ExecutionHookImpl(config, dataObjectMap, plan, currentExecutors));
    ex.execute();
    ex.closeExecution();
}

Example 4 with IExecutor

use of edu.iu.dsc.tws.api.compute.executor.IExecutor in project twister2 by DSC-SPIDAL.

the class TaskExecutor method createExecution.

/**
 * Execute a plan and a graph. This call blocks until the execution finishes. In case of
 * streaming, this call doesn't return while for batch computations it returns after
 * the execution is done.
 *
 * @param graph the dataflow graph
 */
public IExecutor createExecution(ComputeGraph graph) {
    ExecutionPlan plan = plan(graph);
    IExecutor ex = executor.getExecutor(config, plan, graph.getOperationMode(), new ExecutionHookImpl(config, dataObjectMap, plan, currentExecutors));
    currentExecutors.add(ex);
    return ex;
}

Example 5 with IExecutor

use of edu.iu.dsc.tws.api.compute.executor.IExecutor in project twister2 by DSC-SPIDAL.

the class SvmSgdAdvancedRunner method executeIterativeTrainingGraph.

/**
 * This method executes the iterative training graph
 * Training is done in parallel depending on the parallelism factor given
 * In this implementation the data loading parallelism and data computing or
 * training parallelism is same. It is the general model to keep them equal. But
 * you can increase the parallelism the way you want. But it is adviced to keep these
 * values equal. Dynamic parallelism in training is not yet tested fully in Twister2 Framework.
 *
 * @return Twister2 DataObject{@literal <double[]>} containing the reduced weight vector
 */
public DataObject<double[]> executeIterativeTrainingGraph() {
    DataObject<double[]> trainedWeight = null;
    dataStreamer = new InputDataStreamer(this.operationMode, svmJobParameters.isDummy(), this.binaryBatchModel);
    iterativeSVMCompute = new IterativeSVMCompute(this.binaryBatchModel, this.operationMode);
    svmReduce = new SVMReduce(this.operationMode);
    trainingBuilder.addSource(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, dataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = trainingBuilder.addCompute(Constants.SimpleGraphConfig.SVM_COMPUTE, iterativeSVMCompute, svmComputeParallelism);
    ComputeConnection svmReduceConnection = trainingBuilder.addCompute(Constants.SimpleGraphConfig.SVM_REDUCE, svmReduce, reduceParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.DATA_EDGE).withDataType(MessageTypes.OBJECT);
    // svmReduceConnection
    // .reduce(Constants.SimpleGraphConfig.SVM_COMPUTE, Constants.SimpleGraphConfig.REDUCE_EDGE,
    // new ReduceAggregator(), DataType.OBJECT);
    svmReduceConnection.allreduce(Constants.SimpleGraphConfig.SVM_COMPUTE).viaEdge(Constants.SimpleGraphConfig.REDUCE_EDGE).withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.OBJECT);
    trainingBuilder.setMode(operationMode);
    ComputeGraph graph = trainingBuilder.build();
    graph.setGraphName("training-graph");
    ExecutionPlan plan = taskExecutor.plan(graph);
    IExecutor ex = taskExecutor.createExecution(graph, plan);
    // iteration is being decoupled from the computation task
    for (int i = 0; i < this.binaryBatchModel.getIterations(); i++) {
        taskExecutor.addInput(graph, plan, Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, Constants.SimpleGraphConfig.INPUT_DATA, trainingData);
        taskExecutor.addInput(graph, plan, Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, Constants.SimpleGraphConfig.INPUT_WEIGHT_VECTOR, inputWeightVector);
        inputWeightVector = taskExecutor.getOutput(graph, plan, Constants.SimpleGraphConfig.SVM_REDUCE);
        ex.execute();
    }
    ex.closeExecution();
    LOG.info("Task Graph Executed !!! ");
    if (workerId == 0) {
        trainedWeight = retrieveWeightVectorFromTaskGraph(graph, plan);
        this.trainedWeightVector = trainedWeight;
    }
    return trainedWeight;
}