Examples with ExecutionPlan edu.iu.dsc.tws.api.compute.executor.ExecutionPlan used on opensource projects

Example 26 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdAdvancedRunner method executeTrainingGraph.

/**
 * This method executes the training graph
 * Training is done in parallel depending on the parallelism factor given
 * In this implementation the data loading parallelism and data computing or
 * training parallelism is same. It is the general model to keep them equal. But
 * you can increase the parallelism the way you want. But it is adviced to keep these
 * values equal. Dynamic parallelism in training is not yet tested fully in Twister2 Framework.
 *
 * @return Twister2 DataObject{@literal <double[]>} containing the reduced weight vector
 */
public DataObject<double[]> executeTrainingGraph() {
    DataObject<double[]> trainedWeight = null;
    dataStreamer = new InputDataStreamer(this.operationMode, svmJobParameters.isDummy(), this.binaryBatchModel);
    svmCompute = new SVMCompute(this.binaryBatchModel, this.operationMode);
    svmReduce = new SVMReduce(this.operationMode);
    trainingBuilder.addSource(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, dataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = trainingBuilder.addCompute(Constants.SimpleGraphConfig.SVM_COMPUTE, svmCompute, svmComputeParallelism);
    ComputeConnection svmReduceConnection = trainingBuilder.addCompute(Constants.SimpleGraphConfig.SVM_REDUCE, svmReduce, reduceParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.DATA_EDGE).withDataType(MessageTypes.OBJECT);
    // svmReduceConnection
    // .reduce(Constants.SimpleGraphConfig.SVM_COMPUTE, Constants.SimpleGraphConfig.REDUCE_EDGE,
    // new ReduceAggregator(), DataType.OBJECT);
    svmReduceConnection.allreduce(Constants.SimpleGraphConfig.SVM_COMPUTE).viaEdge(Constants.SimpleGraphConfig.REDUCE_EDGE).withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.OBJECT);
    trainingBuilder.setMode(operationMode);
    ComputeGraph graph = trainingBuilder.build();
    graph.setGraphName("training-graph");
    ExecutionPlan plan = taskExecutor.plan(graph);
    taskExecutor.addInput(graph, plan, Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, Constants.SimpleGraphConfig.INPUT_DATA, trainingData);
    taskExecutor.addInput(graph, plan, Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, Constants.SimpleGraphConfig.INPUT_WEIGHT_VECTOR, inputWeightVector);
    taskExecutor.execute(graph, plan);
    LOG.info("Task Graph Executed !!! ");
    if (workerId == 0) {
        trainedWeight = retrieveWeightVectorFromTaskGraph(graph, plan);
        this.trainedWeightVector = trainedWeight;
    }
    return trainedWeight;
}

Example 27 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class TaskExecutor method collectData.

/**
 * This method collects all the output from the provided {@link ExecutionPlan}.
 * The partition IDs will be assigned just before adding the partitions to the {@link DataObject}
 */
public static void collectData(Config cfg, ExecutionPlan executionPlan, Map<String, DataObject> dataMap) {
    Map<Integer, INodeInstance> nodes = executionPlan.getNodes();
    Map<String, DataObject> dataObjectMapForPlan = new HashMap<>();
    if (nodes != null) {
        nodes.forEach((taskId, node) -> {
            INode task = node.getNode();
            if (task instanceof Collector) {
                Set<String> collectibleNames = ((Collector) task).getCollectibleNames();
                collectibleNames.forEach(name -> {
                    DataPartition partition = ((Collector) task).get(name);
                    // if this task outs only one partition and user has implemented no arg get() method
                    if (collectibleNames.size() == 1 && partition == null) {
                        partition = ((Collector) task).get();
                    }
                    if (partition != null) {
                        partition.setId(node.getIndex());
                        dataObjectMapForPlan.computeIfAbsent(name, n -> new DataObjectImpl<>(cfg)).addPartition(partition);
                    } else {
                        LOG.warning(String.format("Task index %d  of task %d returned null for data %s", node.getIndex(), node.getId(), name));
                    }
                });
            }
        });
    }
    dataMap.putAll(dataObjectMapForPlan);
}

Example 28 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class CDFWRuntime method handleExecuteMessage.

private boolean handleExecuteMessage(Any msg) {
    ISenderToDriver senderToDriver = JMWorkerAgent.getJMWorkerAgent().getDriverAgent();
    CDFWJobAPI.ExecuteMessage executeMessage;
    ExecutionPlan executionPlan;
    CDFWJobAPI.ExecuteCompletedMessage completedMessage = null;
    try {
        executeMessage = msg.unpack(CDFWJobAPI.ExecuteMessage.class);
        // get the subgraph from the map
        CDFWJobAPI.SubGraph subGraph = executeMessage.getGraph();
        ComputeGraph taskGraph = (ComputeGraph) serializer.deserialize(subGraph.getGraphSerialized().toByteArray());
        if (taskGraph == null) {
            LOG.severe(workerId + " Unable to find the subgraph " + subGraph.getName());
            return true;
        }
        // use the taskexecutor to create the execution plan
        executionPlan = taskExecutor.plan(taskGraph);
        taskExecutor.execute(taskGraph, executionPlan);
        // reuse the task executor execute
        completedMessage = CDFWJobAPI.ExecuteCompletedMessage.newBuilder().setSubgraphName(subGraph.getName()).build();
        if (!senderToDriver.sendToDriver(completedMessage)) {
            LOG.severe("Unable to send the subgraph completed message :" + completedMessage);
        }
    } catch (InvalidProtocolBufferException e) {
        LOG.log(Level.SEVERE, "Unable to unpack received message ", e);
    }
    return false;
}

Example 29 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class IterativeJob method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    IterativeSourceTask g = new IterativeSourceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder graphBuilder = ComputeGraphBuilder.newBuilder(config);
    graphBuilder.addSource("source", g, 4);
    ComputeConnection computeConnection = graphBuilder.addCompute("sink", r, 4);
    computeConnection.partition("source").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph graph = graphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    IExecutor ex = taskExecutor.createExecution(graph, plan);
    for (int i = 0; i < 10; i++) {
        LOG.info("Starting iteration: " + i);
        taskExecutor.addInput(graph, plan, "source", "input", new DataObjectImpl<>(config));
        // this is a blocking call
        ex.execute();
        DataObject<Object> dataSet = taskExecutor.getOutput(graph, plan, "sink");
        DataPartition<Object>[] values = dataSet.getPartitions();
    }
    ex.closeExecution();
}

Example 30 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdRunner method initializeExecute.

/**
 * Initializing the execute method
 */
public void initializeExecute() {
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    this.operationMode = this.svmJobParameters.isStreaming() ? OperationMode.STREAMING : OperationMode.BATCH;
    DataStreamer dataStreamer = new DataStreamer(this.operationMode, svmJobParameters.isDummy(), this.binaryBatchModel);
    SVMCompute svmCompute = new SVMCompute(this.binaryBatchModel, this.operationMode);
    SVMReduce svmReduce = new SVMReduce(this.operationMode);
    builder.addSource(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, dataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_COMPUTE, svmCompute, svmComputeParallelism);
    ComputeConnection svmReduceConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_REDUCE, svmReduce, reduceParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.DATA_EDGE).withDataType(MessageTypes.OBJECT);
    svmReduceConnection.reduce(Constants.SimpleGraphConfig.SVM_COMPUTE).viaEdge(Constants.SimpleGraphConfig.REDUCE_EDGE).withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(operationMode);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    taskExecutor.execute(graph, plan);
    LOG.info("Task Graph Executed !!! ");
    if (operationMode.equals(OperationMode.BATCH)) {
        DataObject<double[]> dataSet = taskExecutor.getOutput(graph, plan, Constants.SimpleGraphConfig.SVM_REDUCE);
        DataPartition<double[]> values = dataSet.getPartitions()[0];
        DataPartitionConsumer<double[]> dataPartitionConsumer = values.getConsumer();
        // LOG.info("Final Receive  : " + dataPartitionConsumer.hasNext());
        while (dataPartitionConsumer.hasNext()) {
            LOG.info("Final Aggregated Values Are:" + Arrays.toString(dataPartitionConsumer.next()));
        }
    }
}