Examples with OneInputStreamOperator org.apache.flink.streaming.api.operators.OneInputStreamOperator used on opensource projects

Example 1 with OneInputStreamOperator

use of org.apache.flink.streaming.api.operators.OneInputStreamOperator in project flink by apache.

the class WindowedStream method createFastTimeOperatorIfValid.

private <R> SingleOutputStreamOperator<R> createFastTimeOperatorIfValid(InternalWindowFunction<Iterable<T>, R, K, W> function, TypeInformation<R> resultType, String functionName) {
    if (windowAssigner.getClass() == SlidingAlignedProcessingTimeWindows.class && trigger == null && evictor == null) {
        SlidingAlignedProcessingTimeWindows timeWindows = (SlidingAlignedProcessingTimeWindows) windowAssigner;
        final long windowLength = timeWindows.getSize();
        final long windowSlide = timeWindows.getSlide();
        String opName = "Fast " + timeWindows + " of " + functionName;
        @SuppressWarnings("unchecked") InternalWindowFunction<Iterable<T>, R, K, TimeWindow> timeWindowFunction = (InternalWindowFunction<Iterable<T>, R, K, TimeWindow>) function;
        OneInputStreamOperator<T, R> op = new AccumulatingProcessingTimeWindowOperator<>(timeWindowFunction, input.getKeySelector(), input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), input.getType().createSerializer(getExecutionEnvironment().getConfig()), windowLength, windowSlide);
        return input.transform(opName, resultType, op);
    } else if (windowAssigner.getClass() == TumblingAlignedProcessingTimeWindows.class && trigger == null && evictor == null) {
        TumblingAlignedProcessingTimeWindows timeWindows = (TumblingAlignedProcessingTimeWindows) windowAssigner;
        final long windowLength = timeWindows.getSize();
        final long windowSlide = timeWindows.getSize();
        String opName = "Fast " + timeWindows + " of " + functionName;
        @SuppressWarnings("unchecked") InternalWindowFunction<Iterable<T>, R, K, TimeWindow> timeWindowFunction = (InternalWindowFunction<Iterable<T>, R, K, TimeWindow>) function;
        OneInputStreamOperator<T, R> op = new AccumulatingProcessingTimeWindowOperator<>(timeWindowFunction, input.getKeySelector(), input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), input.getType().createSerializer(getExecutionEnvironment().getConfig()), windowLength, windowSlide);
        return input.transform(opName, resultType, op);
    }
    return null;
}

Example 2 with OneInputStreamOperator

use of org.apache.flink.streaming.api.operators.OneInputStreamOperator in project flink by apache.

the class WindowedStream method createFastTimeOperatorIfValid.

private <R> SingleOutputStreamOperator<R> createFastTimeOperatorIfValid(ReduceFunction<?> function, TypeInformation<R> resultType, String functionName) {
    if (windowAssigner.getClass() == SlidingAlignedProcessingTimeWindows.class && trigger == null && evictor == null) {
        SlidingAlignedProcessingTimeWindows timeWindows = (SlidingAlignedProcessingTimeWindows) windowAssigner;
        final long windowLength = timeWindows.getSize();
        final long windowSlide = timeWindows.getSlide();
        String opName = "Fast " + timeWindows + " of " + functionName;
        @SuppressWarnings("unchecked") ReduceFunction<T> reducer = (ReduceFunction<T>) function;
        @SuppressWarnings("unchecked") OneInputStreamOperator<T, R> op = (OneInputStreamOperator<T, R>) new AggregatingProcessingTimeWindowOperator<>(reducer, input.getKeySelector(), input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), input.getType().createSerializer(getExecutionEnvironment().getConfig()), windowLength, windowSlide);
        return input.transform(opName, resultType, op);
    } else if (windowAssigner.getClass() == TumblingAlignedProcessingTimeWindows.class && trigger == null && evictor == null) {
        TumblingAlignedProcessingTimeWindows timeWindows = (TumblingAlignedProcessingTimeWindows) windowAssigner;
        final long windowLength = timeWindows.getSize();
        final long windowSlide = timeWindows.getSize();
        String opName = "Fast " + timeWindows + " of " + functionName;
        @SuppressWarnings("unchecked") ReduceFunction<T> reducer = (ReduceFunction<T>) function;
        @SuppressWarnings("unchecked") OneInputStreamOperator<T, R> op = (OneInputStreamOperator<T, R>) new AggregatingProcessingTimeWindowOperator<>(reducer, input.getKeySelector(), input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), input.getType().createSerializer(getExecutionEnvironment().getConfig()), windowLength, windowSlide);
        return input.transform(opName, resultType, op);
    }
    return null;
}

Example 3 with OneInputStreamOperator

use of org.apache.flink.streaming.api.operators.OneInputStreamOperator in project flink by apache.

the class SideOutputITCase method testWatermarkForwarding.

/**
	 * Verify that watermarks are forwarded to all side outputs.
	 */
@Test
public void testWatermarkForwarding() throws Exception {
    final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side") {
    };
    final OutputTag<String> sideOutputTag2 = new OutputTag<String>("other-side") {
    };
    TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
    TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
    TestListResultSink<String> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(3);
    DataStream<Integer> dataStream = env.addSource(new SourceFunction<Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
            ctx.collectWithTimestamp(1, 0);
            ctx.emitWatermark(new Watermark(0));
            ctx.collectWithTimestamp(2, 1);
            ctx.collectWithTimestamp(5, 2);
            ctx.emitWatermark(new Watermark(2));
            ctx.collectWithTimestamp(3, 3);
            ctx.collectWithTimestamp(4, 4);
        }

        @Override
        public void cancel() {
        }
    });
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
        }
    });
    class WatermarkReifier extends AbstractStreamOperator<String> implements OneInputStreamOperator<String, String> {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(StreamRecord<String> element) throws Exception {
            output.collect(new StreamRecord<>("E:" + element.getValue()));
        }

        @Override
        public void processWatermark(Watermark mark) throws Exception {
            super.processWatermark(mark);
            output.collect(new StreamRecord<>("WM:" + mark.getTimestamp()));
        }
    }
    passThroughtStream.getSideOutput(sideOutputTag1).transform("ReifyWatermarks", BasicTypeInfo.STRING_TYPE_INFO, new WatermarkReifier()).addSink(sideOutputResultSink1);
    passThroughtStream.getSideOutput(sideOutputTag2).transform("ReifyWatermarks", BasicTypeInfo.STRING_TYPE_INFO, new WatermarkReifier()).addSink(sideOutputResultSink2);
    passThroughtStream.map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return value.toString();
        }
    }).transform("ReifyWatermarks", BasicTypeInfo.STRING_TYPE_INFO, new WatermarkReifier()).addSink(resultSink);
    env.execute();
    assertEquals(Arrays.asList("E:sideout-1", "E:sideout-2", "E:sideout-3", "E:sideout-4", "E:sideout-5", "WM:0", "WM:2", "WM:" + Long.MAX_VALUE), sideOutputResultSink1.getSortedResult());
    assertEquals(Arrays.asList("E:sideout-1", "E:sideout-2", "E:sideout-3", "E:sideout-4", "E:sideout-5", "WM:0", "WM:2", "WM:" + Long.MAX_VALUE), sideOutputResultSink1.getSortedResult());
    assertEquals(Arrays.asList("E:1", "E:2", "E:3", "E:4", "E:5", "WM:0", "WM:2", "WM:" + Long.MAX_VALUE), resultSink.getSortedResult());
}