Examples with AccumulatingProcessingTimeWindowOperator org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator used on opensource projects

Example 1 with AccumulatingProcessingTimeWindowOperator

use of org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator in project flink by apache.

the class WindowedStream method createFastTimeOperatorIfValid.

private <R> SingleOutputStreamOperator<R> createFastTimeOperatorIfValid(InternalWindowFunction<Iterable<T>, R, K, W> function, TypeInformation<R> resultType, String functionName) {
    if (windowAssigner.getClass() == SlidingAlignedProcessingTimeWindows.class && trigger == null && evictor == null) {
        SlidingAlignedProcessingTimeWindows timeWindows = (SlidingAlignedProcessingTimeWindows) windowAssigner;
        final long windowLength = timeWindows.getSize();
        final long windowSlide = timeWindows.getSlide();
        String opName = "Fast " + timeWindows + " of " + functionName;
        @SuppressWarnings("unchecked") InternalWindowFunction<Iterable<T>, R, K, TimeWindow> timeWindowFunction = (InternalWindowFunction<Iterable<T>, R, K, TimeWindow>) function;
        OneInputStreamOperator<T, R> op = new AccumulatingProcessingTimeWindowOperator<>(timeWindowFunction, input.getKeySelector(), input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), input.getType().createSerializer(getExecutionEnvironment().getConfig()), windowLength, windowSlide);
        return input.transform(opName, resultType, op);
    } else if (windowAssigner.getClass() == TumblingAlignedProcessingTimeWindows.class && trigger == null && evictor == null) {
        TumblingAlignedProcessingTimeWindows timeWindows = (TumblingAlignedProcessingTimeWindows) windowAssigner;
        final long windowLength = timeWindows.getSize();
        final long windowSlide = timeWindows.getSize();
        String opName = "Fast " + timeWindows + " of " + functionName;
        @SuppressWarnings("unchecked") InternalWindowFunction<Iterable<T>, R, K, TimeWindow> timeWindowFunction = (InternalWindowFunction<Iterable<T>, R, K, TimeWindow>) function;
        OneInputStreamOperator<T, R> op = new AccumulatingProcessingTimeWindowOperator<>(timeWindowFunction, input.getKeySelector(), input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), input.getType().createSerializer(getExecutionEnvironment().getConfig()), windowLength, windowSlide);
        return input.transform(opName, resultType, op);
    }
    return null;
}

Example 2 with AccumulatingProcessingTimeWindowOperator

use of org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator in project flink by apache.

the class FoldApplyProcessWindowFunctionTest method testFoldAllWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldAllWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyProcessAllWindowFunction<TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyProcessAllWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new ProcessAllWindowFunction<Integer, Integer, TimeWindow>() {

        @Override
        public void process(Context context, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Byte, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableProcessAllWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Byte>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Byte getKey(Integer value) throws Exception {
            return 0;
        }
    }, ByteSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.process(foldWindowFunction.new Context() {

        @Override
        public TimeWindow window() {
            return new TimeWindow(0, 1);
        }
    }, input, new ListCollector<>(result));
    Assert.assertEquals(expected, result);
}

Example 3 with AccumulatingProcessingTimeWindowOperator

use of org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator in project flink by apache.

the class FoldApplyWindowFunctionTest method testFoldWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyWindowFunction<Integer, TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        private static final long serialVersionUID = -4849549768529720587L;

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new WindowFunction<Integer, Integer, Integer, TimeWindow>() {

        @Override
        public void apply(Integer integer, TimeWindow window, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Integer>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, IntSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.apply(0, new TimeWindow(0, 1), input, new ListCollector<Integer>(result));
    Assert.assertEquals(expected, result);
}

Example 4 with AccumulatingProcessingTimeWindowOperator

use of org.apache.flink.streaming.runtime.operators.windowing.AccumulatingProcessingTimeWindowOperator in project flink by apache.

the class FoldApplyProcessWindowFunctionTest method testFoldWindowFunctionOutputTypeConfigurable.

/**
	 * Tests that the FoldWindowFunction gets the output type serializer set by the
	 * StreamGraphGenerator and checks that the FoldWindowFunction computes the correct result.
	 */
@Test
public void testFoldWindowFunctionOutputTypeConfigurable() throws Exception {
    StreamExecutionEnvironment env = new DummyStreamExecutionEnvironment();
    List<StreamTransformation<?>> transformations = new ArrayList<>();
    int initValue = 1;
    FoldApplyProcessWindowFunction<Integer, TimeWindow, Integer, Integer, Integer> foldWindowFunction = new FoldApplyProcessWindowFunction<>(initValue, new FoldFunction<Integer, Integer>() {

        @Override
        public Integer fold(Integer accumulator, Integer value) throws Exception {
            return accumulator + value;
        }
    }, new ProcessWindowFunction<Integer, Integer, Integer, TimeWindow>() {

        @Override
        public void process(Integer integer, Context context, Iterable<Integer> input, Collector<Integer> out) throws Exception {
            for (Integer in : input) {
                out.collect(in);
            }
        }
    }, BasicTypeInfo.INT_TYPE_INFO);
    AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> windowOperator = new AccumulatingProcessingTimeWindowOperator<>(new InternalIterableProcessWindowFunction<>(foldWindowFunction), new KeySelector<Integer, Integer>() {

        private static final long serialVersionUID = -7951310554369722809L;

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }, IntSerializer.INSTANCE, IntSerializer.INSTANCE, 3000, 3000);
    SourceFunction<Integer> sourceFunction = new SourceFunction<Integer>() {

        private static final long serialVersionUID = 8297735565464653028L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
        }

        @Override
        public void cancel() {
        }
    };
    SourceTransformation<Integer> source = new SourceTransformation<>("", new StreamSource<>(sourceFunction), BasicTypeInfo.INT_TYPE_INFO, 1);
    transformations.add(new OneInputTransformation<>(source, "test", windowOperator, BasicTypeInfo.INT_TYPE_INFO, 1));
    StreamGraph streamGraph = StreamGraphGenerator.generate(env, transformations, 1);
    List<Integer> result = new ArrayList<>();
    List<Integer> input = new ArrayList<>();
    List<Integer> expected = new ArrayList<>();
    input.add(1);
    input.add(2);
    input.add(3);
    for (int value : input) {
        initValue += value;
    }
    expected.add(initValue);
    foldWindowFunction.process(0, foldWindowFunction.new Context() {

        @Override
        public TimeWindow window() {
            return new TimeWindow(0, 1);
        }
    }, input, new ListCollector<>(result));
    Assert.assertEquals(expected, result);
}