Examples with DataStreamer edu.iu.dsc.tws.examples.ml.svm.streamer.DataStreamer used on opensource projects

Example 1 with DataStreamer

use of edu.iu.dsc.tws.examples.ml.svm.streamer.DataStreamer in project twister2 by DSC-SPIDAL.

the class SvmSgdRunner method initializeExecute.

/**
 * Initializing the execute method
 */
public void initializeExecute() {
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    this.operationMode = this.svmJobParameters.isStreaming() ? OperationMode.STREAMING : OperationMode.BATCH;
    DataStreamer dataStreamer = new DataStreamer(this.operationMode, svmJobParameters.isDummy(), this.binaryBatchModel);
    SVMCompute svmCompute = new SVMCompute(this.binaryBatchModel, this.operationMode);
    SVMReduce svmReduce = new SVMReduce(this.operationMode);
    builder.addSource(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, dataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_COMPUTE, svmCompute, svmComputeParallelism);
    ComputeConnection svmReduceConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_REDUCE, svmReduce, reduceParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.DATA_EDGE).withDataType(MessageTypes.OBJECT);
    svmReduceConnection.reduce(Constants.SimpleGraphConfig.SVM_COMPUTE).viaEdge(Constants.SimpleGraphConfig.REDUCE_EDGE).withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(operationMode);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    taskExecutor.execute(graph, plan);
    LOG.info("Task Graph Executed !!! ");
    if (operationMode.equals(OperationMode.BATCH)) {
        DataObject<double[]> dataSet = taskExecutor.getOutput(graph, plan, Constants.SimpleGraphConfig.SVM_REDUCE);
        DataPartition<double[]> values = dataSet.getPartitions()[0];
        DataPartitionConsumer<double[]> dataPartitionConsumer = values.getConsumer();
        // LOG.info("Final Receive  : " + dataPartitionConsumer.hasNext());
        while (dataPartitionConsumer.hasNext()) {
            LOG.info("Final Aggregated Values Are:" + Arrays.toString(dataPartitionConsumer.next()));
        }
    }
}