Examples with TestListResultSink org.apache.flink.test.streaming.runtime.util.TestListResultSink used on opensource projects

Example 11 with TestListResultSink

use of org.apache.flink.test.streaming.runtime.util.TestListResultSink in project flink by apache.

the class SideOutputITCase method testProcessFunctionSideOutput.

/**
	 * Test ProcessFunction side output.
	 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
    final OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
    passThroughtStream.addSink(resultSink);
    see.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}

Example 12 with TestListResultSink

use of org.apache.flink.test.streaming.runtime.util.TestListResultSink in project flink by apache.

the class SideOutputITCase method testKeyedWindowLateArrivingEvents.

@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
    TestListResultSink<String> resultSink = new TestListResultSink<>();
    TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late") {
    };
    SingleOutputStreamOperator<String> windowOperator = dataStream.assignTimestampsAndWatermarks(new TestWatermarkAssigner()).keyBy(new TestKeySelector()).timeWindow(Time.milliseconds(1), Time.milliseconds(1)).allowedLateness(Time.milliseconds(2)).sideOutputLateData(lateDataTag).apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
            for (Integer val : input) {
                out.collect(String.valueOf(key) + "-" + String.valueOf(val));
            }
        }
    });
    windowOperator.addSink(resultSink);
    windowOperator.getSideOutput(lateDataTag).addSink(lateResultSink);
    see.execute();
    assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
    assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}

Example 13 with TestListResultSink

use of org.apache.flink.test.streaming.runtime.util.TestListResultSink in project flink by apache.

the class SideOutputITCase method testWatermarkForwarding.

/**
	 * Verify that watermarks are forwarded to all side outputs.
	 */
@Test
public void testWatermarkForwarding() throws Exception {
    final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side") {
    };
    final OutputTag<String> sideOutputTag2 = new OutputTag<String>("other-side") {
    };
    TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
    TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
    TestListResultSink<String> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(3);
    DataStream<Integer> dataStream = env.addSource(new SourceFunction<Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void run(SourceContext<Integer> ctx) throws Exception {
            ctx.collectWithTimestamp(1, 0);
            ctx.emitWatermark(new Watermark(0));
            ctx.collectWithTimestamp(2, 1);
            ctx.collectWithTimestamp(5, 2);
            ctx.emitWatermark(new Watermark(2));
            ctx.collectWithTimestamp(3, 3);
            ctx.collectWithTimestamp(4, 4);
        }

        @Override
        public void cancel() {
        }
    });
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
        }
    });
    class WatermarkReifier extends AbstractStreamOperator<String> implements OneInputStreamOperator<String, String> {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(StreamRecord<String> element) throws Exception {
            output.collect(new StreamRecord<>("E:" + element.getValue()));
        }

        @Override
        public void processWatermark(Watermark mark) throws Exception {
            super.processWatermark(mark);
            output.collect(new StreamRecord<>("WM:" + mark.getTimestamp()));
        }
    }
    passThroughtStream.getSideOutput(sideOutputTag1).transform("ReifyWatermarks", BasicTypeInfo.STRING_TYPE_INFO, new WatermarkReifier()).addSink(sideOutputResultSink1);
    passThroughtStream.getSideOutput(sideOutputTag2).transform("ReifyWatermarks", BasicTypeInfo.STRING_TYPE_INFO, new WatermarkReifier()).addSink(sideOutputResultSink2);
    passThroughtStream.map(new MapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public String map(Integer value) throws Exception {
            return value.toString();
        }
    }).transform("ReifyWatermarks", BasicTypeInfo.STRING_TYPE_INFO, new WatermarkReifier()).addSink(resultSink);
    env.execute();
    assertEquals(Arrays.asList("E:sideout-1", "E:sideout-2", "E:sideout-3", "E:sideout-4", "E:sideout-5", "WM:0", "WM:2", "WM:" + Long.MAX_VALUE), sideOutputResultSink1.getSortedResult());
    assertEquals(Arrays.asList("E:sideout-1", "E:sideout-2", "E:sideout-3", "E:sideout-4", "E:sideout-5", "WM:0", "WM:2", "WM:" + Long.MAX_VALUE), sideOutputResultSink1.getSortedResult());
    assertEquals(Arrays.asList("E:1", "E:2", "E:3", "E:4", "E:5", "WM:0", "WM:2", "WM:" + Long.MAX_VALUE), resultSink.getSortedResult());
}

Example 14 with TestListResultSink

use of org.apache.flink.test.streaming.runtime.util.TestListResultSink in project flink by apache.

the class SideOutputITCase method testAllWindowLateArrivingEvents.

/**
	 * Test window late arriving events stream
	 */
@Test
public void testAllWindowLateArrivingEvents() throws Exception {
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(1);
    see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late") {
    };
    SingleOutputStreamOperator<Integer> windowOperator = dataStream.assignTimestampsAndWatermarks(new TestWatermarkAssigner()).timeWindowAll(Time.milliseconds(1), Time.milliseconds(1)).sideOutputLateData(lateDataTag).apply(new AllWindowFunction<Integer, Integer, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(TimeWindow window, Iterable<Integer> values, Collector<Integer> out) throws Exception {
            for (Integer val : values) {
                out.collect(val);
            }
        }
    });
    windowOperator.getSideOutput(lateDataTag).flatMap(new FlatMapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void flatMap(Integer value, Collector<String> out) throws Exception {
            out.collect("late-" + String.valueOf(value));
        }
    }).addSink(sideOutputResultSink);
    see.execute();
    assertEquals(sideOutputResultSink.getSortedResult(), Arrays.asList("late-3", "late-4"));
}

Example 15 with TestListResultSink

use of org.apache.flink.test.streaming.runtime.util.TestListResultSink in project flink by apache.

the class DirectedOutputITCase method outputSelectorTest.

@Test
public void outputSelectorTest() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    TestListResultSink<Long> evenSink = new TestListResultSink<Long>();
    TestListResultSink<Long> oddAndTenSink = new TestListResultSink<Long>();
    TestListResultSink<Long> evenAndOddSink = new TestListResultSink<Long>();
    TestListResultSink<Long> allSink = new TestListResultSink<Long>();
    SplitStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector());
    source.select(EVEN).addSink(evenSink);
    source.select(ODD, TEN).addSink(oddAndTenSink);
    source.select(EVEN, ODD).addSink(evenAndOddSink);
    source.addSink(allSink);
    env.execute();
    assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), evenSink.getSortedResult());
    assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), oddAndTenSink.getSortedResult());
    assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L), evenAndOddSink.getSortedResult());
    assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L), allSink.getSortedResult());
}