Examples with BaseWindowedSink edu.iu.dsc.tws.task.window.core.BaseWindowedSink used on opensource projects

Example 1 with BaseWindowedSink

use of edu.iu.dsc.tws.task.window.core.BaseWindowedSink in project twister2 by DSC-SPIDAL.

the class STWindowEventTimeExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    String edge = "edge";
    BaseWindowSource g = new SourceWindowTimeStampTask(edge);
    ITimestampExtractor<EventTimeData> timestampExtractor = new EventTimeExtractor();
    // Tumbling Window
    BaseWindowedSink dw = new DirectWindowedReceivingTask().withTumblingCountWindow(5);
    BaseWindowedSink dwDuration = new DirectWindowedReceivingTask().withTumblingDurationWindow(2, TimeUnit.MILLISECONDS);
    // Sliding Window
    BaseWindowedSink sdw = new DirectWindowedReceivingTask().withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwDuration = new DirectWindowedReceivingTask().withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwDurationReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwCountSlidingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwCountTumblingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingAggregate = new STWindowExample.DirectAggregateWindowedTask(new AggregateFunctionImpl(1, 2)).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingFold = new DirectFoldWindowedTask(new FoldFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingProcess = new DirectProcessWindowedIntTask(new ProcessFunctionIntImpl()).withCustomTimestampExtractor(timestampExtractor).withAllowedLateness(0, TimeUnit.MILLISECONDS).withWatermarkInterval(1, TimeUnit.MILLISECONDS).withTumblingDurationWindow(1, TimeUnit.MILLISECONDS);
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, sdwCountTumblingProcess, sinkParallelism);
    computeConnection.direct(SOURCE).viaEdge(edge).withDataType(MessageTypes.INTEGER_ARRAY);
    return computeGraphBuilder;
}

Example 2 with BaseWindowedSink

use of edu.iu.dsc.tws.task.window.core.BaseWindowedSink in project twister2 by DSC-SPIDAL.

the class STWindowExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    String edge = "edge";
    BaseWindowSource g = new SourceWindowTask(edge);
    // Tumbling Window
    BaseWindowedSink dw = new DirectWindowedReceivingTask().withTumblingCountWindow(5);
    BaseWindowedSink dwDuration = new DirectWindowedReceivingTask().withTumblingDurationWindow(2, TimeUnit.MILLISECONDS);
    // Sliding Window
    BaseWindowedSink sdw = new DirectWindowedReceivingTask().withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwDuration = new DirectWindowedReceivingTask().withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwDurationReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwCountSlidingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwCountTumblingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingAggregate = new DirectAggregateWindowedTask(new AggregateFunctionImpl(1, 2)).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingFold = new DirectFoldWindowedTask(new FoldFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingProcess = new DirectProcessWindowedTask(new ProcessFunctionImpl()).withSlidingDurationWindow(5, TimeUnit.MILLISECONDS, 3, TimeUnit.MILLISECONDS);
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, sdwCountTumblingProcess, sinkParallelism);
    computeConnection.direct(SOURCE).viaEdge(edge).withDataType(MessageTypes.INTEGER_ARRAY);
    return computeGraphBuilder;
}

Example 3 with BaseWindowedSink

use of edu.iu.dsc.tws.task.window.core.BaseWindowedSink in project twister2 by DSC-SPIDAL.

the class SvmSgdOnlineRunner method buildStreamingTrainingTG.

private ComputeGraph buildStreamingTrainingTG() {
    iterativeStreamingDataStreamer = new IterativeStreamingDataStreamer(this.svmJobParameters.getFeatures(), OperationMode.STREAMING, this.svmJobParameters.isDummy(), this.binaryBatchModel);
    BaseWindowedSink baseWindowedSink = getWindowSinkInstance();
    iterativeStreamingCompute = new IterativeStreamingCompute(OperationMode.STREAMING, new ReduceAggregator(), this.svmJobParameters);
    IterativeStreamingSinkEvaluator iterativeStreamingSinkEvaluator = new IterativeStreamingSinkEvaluator();
    trainingBuilder.addSource(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_DATASTREAMER_SOURCE, iterativeStreamingDataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = trainingBuilder.addCompute(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_SVM_COMPUTE, baseWindowedSink, dataStreamerParallelism);
    ComputeConnection svmReduceConnection = trainingBuilder.addCompute("window-sink", iterativeStreamingCompute, dataStreamerParallelism);
    ComputeConnection svmFinalEvaluationConnection = trainingBuilder.addCompute("window-evaluation-sink", iterativeStreamingSinkEvaluator, dataStreamerParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.STREAMING_EDGE).withDataType(MessageTypes.DOUBLE_ARRAY);
    svmReduceConnection.allreduce(Constants.SimpleGraphConfig.ITERATIVE_STREAMING_SVM_COMPUTE).viaEdge("window-sink-edge").withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.DOUBLE_ARRAY);
    svmFinalEvaluationConnection.allreduce("window-sink").viaEdge("window-evaluation-edge").withReductionFunction(new IterativeAccuracyReduceFunction()).withDataType(MessageTypes.DOUBLE);
    trainingBuilder.setMode(OperationMode.STREAMING);
    trainingBuilder.setTaskGraphName(IterativeSVMConstants.ITERATIVE_STREAMING_TRAINING_TASK_GRAPH);
    return trainingBuilder.build();
}

Example 4 with BaseWindowedSink

use of edu.iu.dsc.tws.task.window.core.BaseWindowedSink in project twister2 by DSC-SPIDAL.

the class SvmSgdOnlineRunner method getWindowSinkInstance.

private BaseWindowedSink getWindowSinkInstance() {
    BaseWindowedSink baseWindowedSink = new IterativeStreamingWindowedCompute(new ProcessWindowFunctionImpl(), OperationMode.STREAMING, this.svmJobParameters, this.binaryBatchModel, "online-training-graph");
    WindowArguments windowArguments = this.svmJobParameters.getWindowArguments();
    TimeUnit timeUnit = TimeUnit.MICROSECONDS;
    if (windowArguments != null) {
        WindowType windowType = windowArguments.getWindowType();
        if (windowArguments.isDuration()) {
            if (windowType.equals(WindowType.TUMBLING)) {
                baseWindowedSink.withTumblingDurationWindow(windowArguments.getWindowLength(), timeUnit);
            }
            if (windowType.equals(WindowType.SLIDING)) {
                baseWindowedSink.withSlidingDurationWindow(windowArguments.getWindowLength(), timeUnit, windowArguments.getSlidingLength(), timeUnit);
            }
        } else {
            if (windowType.equals(WindowType.TUMBLING)) {
                baseWindowedSink.withTumblingCountWindow(windowArguments.getWindowLength());
            }
            if (windowType.equals(WindowType.SLIDING)) {
                baseWindowedSink.withSlidingCountWindow(windowArguments.getWindowLength(), windowArguments.getSlidingLength());
            }
        }
    }
    return baseWindowedSink;
}

Example 5 with BaseWindowedSink

use of edu.iu.dsc.tws.task.window.core.BaseWindowedSink in project twister2 by DSC-SPIDAL.

the class STWindowCustomExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    String edge = "edge";
    BaseWindowSource g = new SourceWindowTask(edge);
    BaseWindowedSink dw = new DirectCustomWindowReceiver().withWindow(TumblingCountWindow.of(5));
    BaseWindowedSink dwDuration = new DirectCustomWindowReceiver().withWindow(TumblingDurationWindow.of(2));
    BaseWindowedSink sdw = new DirectCustomWindowReceiver().withWindow(SlidingCountWindow.of(5, 2));
    WindowConfig.Duration windowLength = new WindowConfig.Duration(2, TimeUnit.MILLISECONDS);
    WindowConfig.Duration slidingLength = new WindowConfig.Duration(2, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwDuration = new DirectCustomWindowReceiver().withWindow(SlidingDurationWindow.of(windowLength, slidingLength));
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, sdwDuration, sinkParallelism);
    computeConnection.direct(SOURCE).viaEdge(edge).withDataType(MessageTypes.INTEGER);
    return computeGraphBuilder;
}