Examples with OutputTag org.apache.flink.util.OutputTag used on opensource projects

Example 71 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by apache.

the class SideOutputITCase method testKeyedCoProcessFunctionSideOutput.

/**
 * Test keyed KeyedCoProcessFunction side output.
 */
@Test
public void testKeyedCoProcessFunctionSideOutput() throws Exception {
    final OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> ds1 = see.fromCollection(elements);
    DataStream<Integer> ds2 = see.fromCollection(elements);
    SingleOutputStreamOperator<Integer> passThroughtStream = ds1.keyBy(i -> i).connect(ds2.keyBy(i -> i)).process(new KeyedCoProcessFunction<Integer, Integer, Integer, Integer>() {

        @Override
        public void processElement1(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            if (value < 3) {
                out.collect(value);
                ctx.output(sideOutputTag, "sideout1-" + ctx.getCurrentKey() + "-" + String.valueOf(value));
            }
        }

        @Override
        public void processElement2(Integer value, Context ctx, Collector<Integer> out) throws Exception {
            if (value >= 3) {
                out.collect(value);
                ctx.output(sideOutputTag, "sideout2-" + ctx.getCurrentKey() + "-" + String.valueOf(value));
            }
        }
    });
    passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
    passThroughtStream.addSink(resultSink);
    see.execute();
    assertEquals(Arrays.asList("sideout1-1-1", "sideout1-2-2", "sideout2-3-3", "sideout2-4-4", "sideout2-5-5"), sideOutputResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}

Example 72 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by apache.

the class SideOutputITCase method testProcessdWindowFunctionSideOutput.

@Test
public void testProcessdWindowFunctionSideOutput() throws Exception {
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    SingleOutputStreamOperator<Integer> windowOperator = dataStream.assignTimestampsAndWatermarks(new TestWatermarkAssigner()).keyBy(new TestKeySelector()).window(SlidingEventTimeWindows.of(Time.milliseconds(1), Time.milliseconds(1))).process(new ProcessWindowFunction<Integer, Integer, Integer, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void process(Integer integer, Context context, Iterable<Integer> elements, Collector<Integer> out) throws Exception {
            out.collect(integer);
            context.output(sideOutputTag, "sideout-" + String.valueOf(integer));
        }
    });
    windowOperator.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
    windowOperator.addSink(resultSink);
    see.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-5"), sideOutputResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 5), resultSink.getSortedResult());
}

Example 73 with OutputTag

use of org.apache.flink.util.OutputTag in project flink-mirror by flink-ci.

the class CEPOperatorTest method testCEPOperatorSideOutputLateElementsEventTime.

@Test
public void testCEPOperatorSideOutputLateElementsEventTime() throws Exception {
    Event startEvent = new Event(41, "c", 1.0);
    Event middle1Event1 = new Event(41, "a", 2.0);
    Event middle1Event2 = new Event(41, "a", 3.0);
    Event middle1Event3 = new Event(41, "a", 4.0);
    OutputTag<Event> lateDataTag = new OutputTag<Event>("late-data", TypeInformation.of(Event.class));
    CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOperator(false, new ComplexNFAFactory(), null, lateDataTag);
    try (OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator)) {
        harness.open();
        harness.processWatermark(new Watermark(Long.MIN_VALUE));
        harness.processElement(new StreamRecord<>(startEvent, 6));
        verifyWatermark(harness.getOutput().poll(), Long.MIN_VALUE);
        harness.processWatermark(new Watermark(6L));
        verifyWatermark(harness.getOutput().poll(), 6L);
        harness.processElement(new StreamRecord<>(middle1Event1, 4));
        harness.processElement(new StreamRecord<>(middle1Event2, 5));
        harness.processElement(new StreamRecord<>(middle1Event3, 7));
        List<Event> late = new ArrayList<>();
        while (!harness.getSideOutput(lateDataTag).isEmpty()) {
            StreamRecord<Event> eventStreamRecord = harness.getSideOutput(lateDataTag).poll();
            late.add(eventStreamRecord.getValue());
        }
        List<Event> expected = Lists.newArrayList(middle1Event1, middle1Event2);
        Assert.assertArrayEquals(expected.toArray(), late.toArray());
    }
}

Example 74 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by splunk.

the class SortingBoundedInputITCase method testBatchExecutionWithTimersOneInput.

@Test
public void testBatchExecutionWithTimersOneInput() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // set parallelism to 1 to have consistent order of results
    env.setParallelism(1);
    Configuration config = new Configuration();
    config.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH);
    env.configure(config, this.getClass().getClassLoader());
    WatermarkStrategy<Tuple2<Integer, Integer>> watermarkStrategy = WatermarkStrategy.forGenerator(ctx -> GENERATE_WATERMARK_AFTER_4_14_TIMESTAMP).withTimestampAssigner((r, previousTimestamp) -> r.f1);
    SingleOutputStreamOperator<Tuple2<Integer, Integer>> elements = env.fromElements(Tuple2.of(1, 3), Tuple2.of(1, 1), Tuple2.of(2, 1), Tuple2.of(1, 4), // late element
    Tuple2.of(2, 3), // late element
    Tuple2.of(1, 2), Tuple2.of(1, 13), Tuple2.of(1, 11), Tuple2.of(2, 14), // late element
    Tuple2.of(1, 11)).assignTimestampsAndWatermarks(watermarkStrategy);
    OutputTag<Integer> lateElements = new OutputTag<>("late_elements", BasicTypeInfo.INT_TYPE_INFO);
    SingleOutputStreamOperator<Tuple3<Long, Integer, Integer>> sums = elements.map(element -> element.f0).keyBy(element -> element).process(new KeyedProcessFunction<Integer, Integer, Tuple3<Long, Integer, Integer>>() {

        private MapState<Long, Integer> countState;

        private ValueState<Long> previousTimestampState;

        @Override
        public void open(Configuration parameters) {
            countState = getRuntimeContext().getMapState(new MapStateDescriptor<>("sum", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO));
            previousTimestampState = getRuntimeContext().getState(new ValueStateDescriptor<>("previousTimestamp", BasicTypeInfo.LONG_TYPE_INFO));
        }

        @Override
        public void processElement(Integer value, Context ctx, Collector<Tuple3<Long, Integer, Integer>> out) throws Exception {
            Long elementTimestamp = ctx.timestamp();
            long nextTen = ((elementTimestamp + 10) / 10) * 10;
            ctx.timerService().registerEventTimeTimer(nextTen);
            if (elementTimestamp < ctx.timerService().currentWatermark()) {
                ctx.output(lateElements, value);
            } else {
                Long previousTimestamp = Optional.ofNullable(previousTimestampState.value()).orElse(0L);
                assertThat(elementTimestamp, greaterThanOrEqualTo(previousTimestamp));
                previousTimestampState.update(elementTimestamp);
                Integer currentCount = Optional.ofNullable(countState.get(nextTen)).orElse(0);
                countState.put(nextTen, currentCount + 1);
            }
        }

        @Override
        public void onTimer(long timestamp, OnTimerContext ctx, Collector<Tuple3<Long, Integer, Integer>> out) throws Exception {
            out.collect(Tuple3.of(timestamp, ctx.getCurrentKey(), countState.get(timestamp)));
            countState.remove(timestamp);
            // this would go in infinite loop if we did not quiesce the
            // timer service.
            ctx.timerService().registerEventTimeTimer(timestamp + 1);
        }
    });
    DataStream<Integer> lateStream = sums.getSideOutput(lateElements);
    List<Integer> lateRecordsCollected = CollectionUtil.iteratorToList(DataStreamUtils.collect(lateStream));
    List<Tuple3<Long, Integer, Integer>> sumsCollected = CollectionUtil.iteratorToList(DataStreamUtils.collect(sums));
    assertTrue(lateRecordsCollected.isEmpty());
    assertThat(sumsCollected, equalTo(Arrays.asList(Tuple3.of(10L, 1, 4), Tuple3.of(20L, 1, 3), Tuple3.of(10L, 2, 2), Tuple3.of(20L, 2, 1))));
}

Example 75 with OutputTag

use of org.apache.flink.util.OutputTag in project flink by splunk.

the class SideOutputITCase method testProcessdWindowFunctionSideOutput.

@Test
public void testProcessdWindowFunctionSideOutput() throws Exception {
    TestListResultSink<Integer> resultSink = new TestListResultSink<>();
    TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
    StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
    see.setParallelism(3);
    DataStream<Integer> dataStream = see.fromCollection(elements);
    OutputTag<String> sideOutputTag = new OutputTag<String>("side") {
    };
    SingleOutputStreamOperator<Integer> windowOperator = dataStream.assignTimestampsAndWatermarks(new TestWatermarkAssigner()).keyBy(new TestKeySelector()).window(SlidingEventTimeWindows.of(Time.milliseconds(1), Time.milliseconds(1))).process(new ProcessWindowFunction<Integer, Integer, Integer, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void process(Integer integer, Context context, Iterable<Integer> elements, Collector<Integer> out) throws Exception {
            out.collect(integer);
            context.output(sideOutputTag, "sideout-" + String.valueOf(integer));
        }
    });
    windowOperator.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
    windowOperator.addSink(resultSink);
    see.execute();
    assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-5"), sideOutputResultSink.getSortedResult());
    assertEquals(Arrays.asList(1, 2, 5), resultSink.getSortedResult());
}