Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 86 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class WindowOperatorTest method testPointSessions.

/**
 * This tests a custom Session window assigner that assigns some elements to "point windows",
 * windows that have the same timestamp for start and end.
 *
 * <p>In this test, elements that have 33 as the second tuple field will be put into a point
 * window.
 */
@Test
@SuppressWarnings("unchecked")
public void testPointSessions() throws Exception {
    closeCalled.set(0);
    WindowOperator operator = WindowOperatorBuilder.builder().withInputFields(inputFieldTypes).withShiftTimezone(shiftTimeZone).assigner(new PointSessionWindowAssigner(3000)).withEventTime(2).aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    // add elements out-of-order
    testHarness.processElement(insertRecord("key2", 1, 0L));
    testHarness.processElement(insertRecord("key2", 33, 1000L));
    // do a snapshot, close and restore again
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    testHarness.close();
    testHarness = createTestHarness(operator);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.processElement(insertRecord("key2", 33, 2500L));
    testHarness.processElement(insertRecord("key1", 1, 10L));
    testHarness.processElement(insertRecord("key1", 2, 1000L));
    testHarness.processElement(insertRecord("key1", 33, 2500L));
    testHarness.processWatermark(new Watermark(12000));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 36L, 3L, localMills(10L), localMills(4000L), localMills(3999L))));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 67L, 3L, localMills(0L), localMills(3000L), localMills(2999L))));
    expectedOutput.add(new Watermark(12000));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
    // we close once in the rest...
    assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 87 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class WindowOperatorTest method testTumblingCountWindow.

@Test
public void testTumblingCountWindow() throws Exception {
    if (!UTC_ZONE_ID.equals(shiftTimeZone)) {
        return;
    }
    closeCalled.set(0);
    final int windowSize = 3;
    LogicalType[] windowTypes = new LogicalType[] { new BigIntType() };
    WindowOperator operator = WindowOperatorBuilder.builder().withInputFields(inputFieldTypes).withShiftTimezone(shiftTimeZone).countWindow(windowSize).aggregateAndBuild(getCountWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(insertRecord("key2", 1, 0L));
    testHarness.processElement(insertRecord("key2", 2, 1000L));
    testHarness.processElement(insertRecord("key2", 3, 2500L));
    testHarness.processElement(insertRecord("key1", 1, 10L));
    testHarness.processElement(insertRecord("key1", 2, 1000L));
    testHarness.processWatermark(new Watermark(12000));
    testHarness.setProcessingTime(12000L);
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 6L, 3L, 0L)));
    expectedOutput.add(new Watermark(12000));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    // do a snapshot, close and restore again
    OperatorSubtaskState snapshotV2 = testHarness.snapshot(0L, 0);
    testHarness.close();
    expectedOutput.clear();
    testHarness = createTestHarness(operator);
    testHarness.setup();
    testHarness.initializeState(snapshotV2);
    testHarness.open();
    testHarness.processElement(insertRecord("key1", 2, 2500L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 5L, 3L, 0L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(insertRecord("key2", 4, 5501L));
    testHarness.processElement(insertRecord("key2", 5, 6000L));
    testHarness.processElement(insertRecord("key2", 5, 6000L));
    testHarness.processElement(insertRecord("key2", 6, 6050L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 14L, 3L, 1L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(insertRecord("key1", 3, 4000L));
    testHarness.processElement(insertRecord("key2", 10, 15000L));
    testHarness.processElement(insertRecord("key2", 20, 15000L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 36L, 3L, 2L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(insertRecord("key1", 2, 2500L));
    testHarness.processElement(insertRecord("key1", 2, 2500L));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 7L, 3L, 1L)));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
    // we close once in the rest...
    assertEquals("Close was not called.", 2, closeCalled.get());
}

Example 88 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class RowTimeRangeUnboundedPrecedingFunctionTest method testLateRecordMetrics.

@Test
public void testLateRecordMetrics() throws Exception {
    RowTimeRangeUnboundedPrecedingFunction<RowData> function = new RowTimeRangeUnboundedPrecedingFunction<>(1000, 2000, aggsHandleFunction, accTypes, inputFieldTypes, 2);
    KeyedProcessOperator<RowData, RowData, RowData> operator = new KeyedProcessOperator<>(function);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    Counter counter = function.getCounter();
    // put some records
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 500L));
    testHarness.processWatermark(new Watermark(500L));
    // late record
    testHarness.processElement(insertRecord("key", 1L, 400L));
    assertEquals(1L, counter.getCount());
}

Example 89 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class IntervalJoinITCase method testBoundedUnorderedStreamsStillJoinCorrectly.

@Test
public void testBoundedUnorderedStreamsStillJoinCorrectly() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    DataStream<Tuple2<String, Integer>> streamOne = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {

        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) {
            ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L);
            ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L);
            ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L);
            ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L);
            ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L);
            ctx.emitWatermark(new Watermark(5));
            ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L);
            ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L);
            ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L);
            ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L);
        }

        @Override
        public void cancel() {
        // do nothing
        }
    });
    DataStream<Tuple2<String, Integer>> streamTwo = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {

        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) {
            ctx.collectWithTimestamp(Tuple2.of("key", 2), 2L);
            ctx.collectWithTimestamp(Tuple2.of("key", 1), 1L);
            ctx.collectWithTimestamp(Tuple2.of("key", 3), 3L);
            ctx.collectWithTimestamp(Tuple2.of("key", 4), 4L);
            ctx.collectWithTimestamp(Tuple2.of("key", 5), 5L);
            ctx.emitWatermark(new Watermark(5));
            ctx.collectWithTimestamp(Tuple2.of("key", 8), 8L);
            ctx.collectWithTimestamp(Tuple2.of("key", 7), 7L);
            ctx.collectWithTimestamp(Tuple2.of("key", 9), 9L);
            ctx.collectWithTimestamp(Tuple2.of("key", 6), 6L);
        }

        @Override
        public void cancel() {
        // do nothing
        }
    });
    streamOne.keyBy(new Tuple2KeyExtractor()).intervalJoin(streamTwo.keyBy(new Tuple2KeyExtractor())).between(Time.milliseconds(-1), Time.milliseconds(1)).process(new CombineToStringJoinFunction()).addSink(new ResultSink());
    env.execute();
    expectInAnyOrder("(key,1):(key,1)", "(key,1):(key,2)", "(key,2):(key,1)", "(key,2):(key,2)", "(key,2):(key,3)", "(key,3):(key,2)", "(key,3):(key,3)", "(key,3):(key,4)", "(key,4):(key,3)", "(key,4):(key,4)", "(key,4):(key,5)", "(key,5):(key,4)", "(key,5):(key,5)", "(key,5):(key,6)", "(key,6):(key,5)", "(key,6):(key,6)", "(key,6):(key,7)", "(key,7):(key,6)", "(key,7):(key,7)", "(key,7):(key,8)", "(key,8):(key,7)", "(key,8):(key,8)", "(key,8):(key,9)", "(key,9):(key,8)", "(key,9):(key,9)");
}

Example 90 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class WindowOperatorTest method testReduceSessionWindowsWithProcessFunction.

@Test
@SuppressWarnings("unchecked")
public void testReduceSessionWindowsWithProcessFunction() throws Exception {
    closeCalled.set(0);
    final int sessionSize = 3;
    ReducingStateDescriptor<Tuple2<String, Integer>> stateDesc = new ReducingStateDescriptor<>("window-contents", new SumReducer(), STRING_INT_TUPLE.createSerializer(new ExecutionConfig()));
    WindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple3<String, Long, Long>, TimeWindow> operator = new WindowOperator<>(EventTimeSessionWindows.withGap(Time.seconds(sessionSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalSingleValueProcessWindowFunction<>(new ReducedProcessSessionWindowFunction()), EventTimeTrigger.create(), 0, null);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple3<String, Long, Long>> testHarness = createTestHarness(operator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    // add elements out-of-order
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 0));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 2), 1000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 3), 2500));
    // do a snapshot, close and restore again
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0L);
    testHarness.close();
    testHarness = createTestHarness(operator);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), 10));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 2), 1000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 3), 2500));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 4), 5501));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 5), 6000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 6), 6050));
    testHarness.processWatermark(new Watermark(12000));
    expectedOutput.add(new StreamRecord<>(new Tuple3<>("key1-6", 10L, 5500L), 5499));
    expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-6", 0L, 5500L), 5499));
    expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-20", 5501L, 9050L), 9049));
    expectedOutput.add(new Watermark(12000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 10), 15000));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 20), 15000));
    testHarness.processWatermark(new Watermark(17999));
    expectedOutput.add(new StreamRecord<>(new Tuple3<>("key2-30", 15000L, 18000L), 17999));
    expectedOutput.add(new Watermark(17999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new Tuple3ResultSortComparator());
    testHarness.close();
}