Examples with BaseWindowSource edu.iu.dsc.tws.task.window.BaseWindowSource used on opensource projects

Example 1 with BaseWindowSource

use of edu.iu.dsc.tws.task.window.BaseWindowSource in project twister2 by DSC-SPIDAL.

the class STWindowEventTimeExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    String edge = "edge";
    BaseWindowSource g = new SourceWindowTimeStampTask(edge);
    ITimestampExtractor<EventTimeData> timestampExtractor = new EventTimeExtractor();
    // Tumbling Window
    BaseWindowedSink dw = new DirectWindowedReceivingTask().withTumblingCountWindow(5);
    BaseWindowedSink dwDuration = new DirectWindowedReceivingTask().withTumblingDurationWindow(2, TimeUnit.MILLISECONDS);
    // Sliding Window
    BaseWindowedSink sdw = new DirectWindowedReceivingTask().withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwDuration = new DirectWindowedReceivingTask().withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwDurationReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwCountSlidingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwCountTumblingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingAggregate = new STWindowExample.DirectAggregateWindowedTask(new AggregateFunctionImpl(1, 2)).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingFold = new DirectFoldWindowedTask(new FoldFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingProcess = new DirectProcessWindowedIntTask(new ProcessFunctionIntImpl()).withCustomTimestampExtractor(timestampExtractor).withAllowedLateness(0, TimeUnit.MILLISECONDS).withWatermarkInterval(1, TimeUnit.MILLISECONDS).withTumblingDurationWindow(1, TimeUnit.MILLISECONDS);
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, sdwCountTumblingProcess, sinkParallelism);
    computeConnection.direct(SOURCE).viaEdge(edge).withDataType(MessageTypes.INTEGER_ARRAY);
    return computeGraphBuilder;
}

Example 2 with BaseWindowSource

use of edu.iu.dsc.tws.task.window.BaseWindowSource in project twister2 by DSC-SPIDAL.

the class STWindowExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    String edge = "edge";
    BaseWindowSource g = new SourceWindowTask(edge);
    // Tumbling Window
    BaseWindowedSink dw = new DirectWindowedReceivingTask().withTumblingCountWindow(5);
    BaseWindowedSink dwDuration = new DirectWindowedReceivingTask().withTumblingDurationWindow(2, TimeUnit.MILLISECONDS);
    // Sliding Window
    BaseWindowedSink sdw = new DirectWindowedReceivingTask().withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwDuration = new DirectWindowedReceivingTask().withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwDurationReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingDurationWindow(2, TimeUnit.MILLISECONDS, 1, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwCountSlidingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withSlidingCountWindow(5, 2);
    BaseWindowedSink sdwCountTumblingReduce = new DirectReduceWindowedTask(new ReduceFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingAggregate = new DirectAggregateWindowedTask(new AggregateFunctionImpl(1, 2)).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingFold = new DirectFoldWindowedTask(new FoldFunctionImpl()).withTumblingCountWindow(5);
    BaseWindowedSink sdwCountTumblingProcess = new DirectProcessWindowedTask(new ProcessFunctionImpl()).withSlidingDurationWindow(5, TimeUnit.MILLISECONDS, 3, TimeUnit.MILLISECONDS);
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, sdwCountTumblingProcess, sinkParallelism);
    computeConnection.direct(SOURCE).viaEdge(edge).withDataType(MessageTypes.INTEGER_ARRAY);
    return computeGraphBuilder;
}

Example 3 with BaseWindowSource

use of edu.iu.dsc.tws.task.window.BaseWindowSource in project twister2 by DSC-SPIDAL.

the class STWindowCustomExample method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    String edge = "edge";
    BaseWindowSource g = new SourceWindowTask(edge);
    BaseWindowedSink dw = new DirectCustomWindowReceiver().withWindow(TumblingCountWindow.of(5));
    BaseWindowedSink dwDuration = new DirectCustomWindowReceiver().withWindow(TumblingDurationWindow.of(2));
    BaseWindowedSink sdw = new DirectCustomWindowReceiver().withWindow(SlidingCountWindow.of(5, 2));
    WindowConfig.Duration windowLength = new WindowConfig.Duration(2, TimeUnit.MILLISECONDS);
    WindowConfig.Duration slidingLength = new WindowConfig.Duration(2, TimeUnit.MILLISECONDS);
    BaseWindowedSink sdwDuration = new DirectCustomWindowReceiver().withWindow(SlidingDurationWindow.of(windowLength, slidingLength));
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, sdwDuration, sinkParallelism);
    computeConnection.direct(SOURCE).viaEdge(edge).withDataType(MessageTypes.INTEGER);
    return computeGraphBuilder;
}

Example 4 with BaseWindowSource

use of edu.iu.dsc.tws.task.window.BaseWindowSource in project twister2 by DSC-SPIDAL.

the class STWindowMPI method buildTaskGraph.

@Override
public ComputeGraphBuilder buildTaskGraph() {
    List<Integer> taskStages = jobParameters.getTaskStages();
    int sourceParallelism = taskStages.get(0);
    int sinkParallelism = taskStages.get(1);
    String edge = "edge";
    BaseWindowSource g = new SourceWindowTask(edge);
    // Tumbling Window
    BaseWindowedSink dw = new DirectWindowedReceivingTask().withTumblingCountWindow(1);
    computeGraphBuilder.addSource(SOURCE, g, sourceParallelism);
    computeConnection = computeGraphBuilder.addCompute(SINK, dw, sinkParallelism);
    computeConnection.direct(SOURCE).viaEdge(edge).withDataType(MessageTypes.INTEGER_ARRAY);
    return computeGraphBuilder;
}