Examples with OutputTag org.apache.flink.util.OutputTag used on opensource projects

Example 76 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by splunk.

the class SideOutputITCase method testProcessFunctionSideOutput.

/**
 * Test ProcessFunction side output.
 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
    final OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
    passThroughtStream.addSink(resultSink);
    see.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}

Example 77 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by splunk.

the class SideOutputITCase method testKeyedProcessFunctionSideOutput.

/**
 * Test keyed ProcessFunction side output.
 */
@Test
public void testKeyedProcessFunctionSideOutput() throws Exception {
    final OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = dataStream.keyBy(new KeySelector<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer getKey(Integer value) throws Exception {
            return value;
        }
    }).process(new ProcessFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            out.collect(value);
            ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
    passThroughtStream.addSink(resultSink);
    see.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}

Example 78 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by splunk.

the class SideOutputITCase method testUnionOfTwoSideOutputs.

@Test
public void testUnionOfTwoSideOutputs() throws Exception {
    TestListResultSink<Integer> evensResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsUEvensResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> evensUOddsResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsUOddsResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> evensUEvensResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> oddsUEvensExternalResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    DataStream<Integer> input = env.fromElements(1, 2, 3, 4);
    OutputTag<Integer> oddTag = new OutputTag<Integer>("odds") {
    };
    OutputTag<Integer> evenTag = new OutputTag<Integer>("even") {
    };
    SingleOutputStreamOperator<Integer> passThroughStream = input.process(new ProcessFunction<Integer, Integer>() {

        @Override
        public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            if (value % 2 != 0) {
                ctx.output(oddTag, value);
            } else {
                ctx.output(evenTag, value);
            }
            out.collect(value);
        }
    });
    DataStream<Integer> evens = passThroughStream.getSideOutput(evenTag);
    DataStream<Integer> odds = passThroughStream.getSideOutput(oddTag);
    evens.addSink(evensResultSink);
    odds.addSink(oddsResultSink);
    passThroughStream.addSink(resultSink);
    odds.union(evens).addSink(oddsUEvensResultSink);
    evens.union(odds).addSink(evensUOddsResultSink);
    odds.union(odds).addSink(oddsUOddsResultSink);
    evens.union(evens).addSink(evensUEvensResultSink);
    odds.union(env.fromElements(2, 4)).addSink(oddsUEvensExternalResultSink);
    env.execute();
    assertEquals(Arrays.asList(1, 3), oddsResultSink.getSortedResult());
    assertEquals(Arrays.asList(2, 4), evensResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), resultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), oddsUEvensResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), evensUOddsResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 1, 3, 3), oddsUOddsResultSink.getSortedResult());
    assertEquals(Arrays.asList(2, 2, 4, 4), evensUEvensResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4), oddsUEvensExternalResultSink.getSortedResult());
}

Example 79 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by splunk.

the class SideOutputITCase method testAllWindowLateArrivingEvents.

/**
 * Test window late arriving events stream.
 */
@Test
public void testAllWindowLateArrivingEvents() throws Exception {
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(1);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late") {
    };
    SingleOutputStreamOperator<Integer> windowOperator = dataStream.assignTimestampsAndWatermarks(new TestWatermarkAssigner()).windowAll(SlidingEventTimeWindows.of(Time.milliseconds(1), Time.milliseconds(1))).sideOutputLateData(lateDataTag).apply(new AllWindowFunction<Integer, Integer, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(TimeWindow window, Iterable<Integer> values, Collector<Integer> out) throws Exception {
            for (Integer val : values) {
                out.collect(val);
            }
        }
    });
    windowOperator.getSideOutput(lateDataTag).flatMap(new FlatMapFunction<Integer, String>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void flatMap(Integer value, Collector<String> out) throws Exception {
            out.collect("late-" + String.valueOf(value));
        }
    }).addSink(sideOutputResultSink);
    see.execute();
    assertEquals(sideOutputResultSink.getSortedResult(), Arrays.asList("late-3", "late-4"));
}

Example 80 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by splunk.

the class SideOutputITCase method testKeyedWindowLateArrivingEvents.

@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
    TestListResultSink<String> resultSink = new TestListResultSink<>();
    TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late") {
    };
    SingleOutputStreamOperator<String> windowOperator = dataStream.assignTimestampsAndWatermarks(new TestWatermarkAssigner()).keyBy(new TestKeySelector()).window(SlidingEventTimeWindows.of(Time.milliseconds(1), Time.milliseconds(1))).allowedLateness(Time.milliseconds(2)).sideOutputLateData(lateDataTag).apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
            for (Integer val : input) {
                out.collect(String.valueOf(key) + "-" + String.valueOf(val));
            }
        }
    });
    windowOperator.addSink(resultSink);
    windowOperator.getSideOutput(lateDataTag).addSink(lateResultSink);
    see.execute();
    assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
    assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}