Examples with IterativeStreamingCompute edu.iu.dsc.tws.examples.ml.svm.compute.IterativeStreamingCompute used on opensource projects

Example 1 with IterativeStreamingCompute

use of edu.iu.dsc.tws.examples.ml.svm.compute.IterativeStreamingCompute in project twister2 by DSC-SPIDAL.

the class SvmSgdOnlineRunner method buildStreamingTrainingTG.

private ComputeGraph buildStreamingTrainingTG() {
    iterativeStreamingDataStreamer = new IterativeStreamingDataStreamer(this.svmJobParameters.getFeatures(), OperationMode.STREAMING, this.svmJobParameters.isDummy(), this.binaryBatchModel);
    BaseWindowedSink baseWindowedSink = getWindowSinkInstance();
    iterativeStreamingCompute = new IterativeStreamingCompute(OperationMode.STREAMING, new ReduceAggregator(), this.svmJobParameters);
    IterativeStreamingSinkEvaluator iterativeStreamingSinkEvaluator = new IterativeStreamingSinkEvaluator();
    trainingBuilder.addSource(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_DATASTREAMER_SOURCE, iterativeStreamingDataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = trainingBuilder.addCompute(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_SVM_COMPUTE, baseWindowedSink, dataStreamerParallelism);
    ComputeConnection svmReduceConnection = trainingBuilder.addCompute("window-sink", iterativeStreamingCompute, dataStreamerParallelism);
    ComputeConnection svmFinalEvaluationConnection = trainingBuilder.addCompute("window-evaluation-sink", iterativeStreamingSinkEvaluator, dataStreamerParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.STREAMING_EDGE).withDataType(MessageTypes.DOUBLE_ARRAY);
    svmReduceConnection.allreduce(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_SVM_COMPUTE).viaEdge("window-sink-edge").withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.DOUBLE_ARRAY);
    svmFinalEvaluationConnection.allreduce("window-sink").viaEdge("window-evaluation-edge").withReductionFunction(new IterativeAccuracyReduceFunction()).withDataType(MessageTypes.DOUBLE);
    trainingBuilder.setMode(OperationMode.STREAMING);
    trainingBuilder.setTaskGraphName(IterativeSVMConstants.ITERATIVE_STREAMING_TRAINING_TASK_GRAPH);
    return trainingBuilder.build();
}