Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 76 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class RowTimeRangeBoundedPrecedingFunctionTest method testLateRecordMetrics.

@Test
public void testLateRecordMetrics() throws Exception {
    RowTimeRangeBoundedPrecedingFunction<RowData> function = new RowTimeRangeBoundedPrecedingFunction<>(aggsHandleFunction, accTypes, inputFieldTypes, 2000, 2);
    KeyedProcessOperator<RowData, RowData, RowData> operator = new KeyedProcessOperator<>(function);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    Counter counter = function.getCounter();
    // put some records
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 500L));
    testHarness.processWatermark(new Watermark(500L));
    // late record
    testHarness.processElement(insertRecord("key", 1L, 400L));
    assertEquals(1L, counter.getCount());
}

Example 77 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class WindowTableFunctionOperatorTest method testTumblingWindows.

@Test
public void testTumblingWindows() throws Exception {
    final TumblingWindowAssigner assigner = TumblingWindowAssigner.of(Duration.ofSeconds(3));
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(assigner, shiftTimeZone);
    testHarness.setup(OUT_SERIALIZER);
    testHarness.open();
    // process elements
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.processElement(insertRecord("key1", 1, 20L));
    testHarness.processElement(insertRecord("key2", 1, 3999L));
    testHarness.processWatermark(new Watermark(999));
    // append 3 fields: window_start, window_end, window_time
    expectedOutput.add(insertRecord("key1", 1, 20L, localMills(0L), localMills(3000L), 2999L));
    expectedOutput.add(insertRecord("key2", 1, 3999L, localMills(3000L), localMills(6000L), 5999L));
    expectedOutput.add(new Watermark(999));
    ASSERTER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    // late element would not be dropped
    testHarness.processElement(insertRecord("key2", 1, 80L));
    expectedOutput.add(insertRecord("key2", 1, 80L, localMills(0L), localMills(3000L), 2999L));
    ASSERTER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 78 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class WindowTableFunctionOperatorTest method testCumulativeWindows.

@Test
public void testCumulativeWindows() throws Exception {
    final CumulativeWindowAssigner assigner = CumulativeWindowAssigner.of(Duration.ofSeconds(3), Duration.ofSeconds(1));
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(assigner, shiftTimeZone);
    testHarness.setup(OUT_SERIALIZER);
    testHarness.open();
    // process elements
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.processElement(insertRecord("key1", 1, 20L));
    testHarness.processElement(insertRecord("key2", 1, 3999L));
    testHarness.processWatermark(new Watermark(999));
    // append 3 fields: window_start, window_end, window_time
    expectedOutput.add(insertRecord("key1", 1, 20L, localMills(0), localMills(1000L), 999L));
    expectedOutput.add(insertRecord("key1", 1, 20L, localMills(0), localMills(2000L), 1999L));
    expectedOutput.add(insertRecord("key1", 1, 20L, localMills(0L), localMills(3000L), 2999L));
    expectedOutput.add(insertRecord("key2", 1, 3999L, localMills(3000L), localMills(4000L), 3999L));
    expectedOutput.add(insertRecord("key2", 1, 3999L, localMills(3000L), localMills(5000L), 4999L));
    expectedOutput.add(insertRecord("key2", 1, 3999L, localMills(3000L), localMills(6000L), 5999L));
    expectedOutput.add(new Watermark(999));
    ASSERTER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    // late element would not be dropped
    testHarness.processElement(insertRecord("key2", 1, 80L));
    expectedOutput.add(insertRecord("key2", 1, 80L, localMills(0), localMills(1000L), 999L));
    expectedOutput.add(insertRecord("key2", 1, 80L, localMills(0), localMills(2000L), 1999L));
    expectedOutput.add(insertRecord("key2", 1, 80L, localMills(0L), localMills(3000L), 2999L));
    ASSERTER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 79 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class RowTimeSortOperatorTest method testSortOnTwoFields.

@Test
public void testSortOnTwoFields() throws Exception {
    InternalTypeInfo<RowData> inputRowType = InternalTypeInfo.ofFields(new IntType(), new BigIntType(), VarCharType.STRING_TYPE, new IntType());
    // Note: RowTimeIdx must be 0 in product environment, the value is 1 here just for simplify
    // the testing
    int rowTimeIdx = 1;
    GeneratedRecordComparator gComparator = new GeneratedRecordComparator("", "", new Object[0]) {

        private static final long serialVersionUID = -6067266199060901331L;

        @Override
        public RecordComparator newInstance(ClassLoader classLoader) {
            return IntRecordComparator.INSTANCE;
        }
    };
    RowDataHarnessAssertor assertor = new RowDataHarnessAssertor(inputRowType.toRowFieldTypes());
    RowTimeSortOperator operator = createSortOperator(inputRowType, rowTimeIdx, gComparator);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    testHarness.processElement(insertRecord(3, 3L, "Hello world", 3));
    testHarness.processElement(insertRecord(2, 2L, "Hello", 2));
    testHarness.processElement(insertRecord(6, 2L, "Luke Skywalker", 6));
    testHarness.processElement(insertRecord(5, 3L, "I am fine.", 5));
    testHarness.processElement(insertRecord(7, 1L, "Comment#1", 7));
    testHarness.processElement(insertRecord(9, 4L, "Comment#3", 9));
    testHarness.processElement(insertRecord(10, 4L, "Comment#4", 10));
    testHarness.processElement(insertRecord(8, 4L, "Comment#2", 8));
    testHarness.processElement(insertRecord(1, 1L, "Hi", 2));
    testHarness.processElement(insertRecord(1, 1L, "Hi", 1));
    testHarness.processElement(insertRecord(4, 3L, "Helloworld, how are you?", 4));
    testHarness.processElement(insertRecord(4, 5L, "Hello, how are you?", 4));
    testHarness.processWatermark(new Watermark(4L));
    List<Object> expectedOutput = new ArrayList<>();
    expectedOutput.add(insertRecord(1, 1L, "Hi", 2));
    expectedOutput.add(insertRecord(1, 1L, "Hi", 1));
    expectedOutput.add(insertRecord(7, 1L, "Comment#1", 7));
    expectedOutput.add(insertRecord(2, 2L, "Hello", 2));
    expectedOutput.add(insertRecord(6, 2L, "Luke Skywalker", 6));
    expectedOutput.add(insertRecord(3, 3L, "Hello world", 3));
    expectedOutput.add(insertRecord(4, 3L, "Helloworld, how are you?", 4));
    expectedOutput.add(insertRecord(5, 3L, "I am fine.", 5));
    expectedOutput.add(insertRecord(8, 4L, "Comment#2", 8));
    expectedOutput.add(insertRecord(9, 4L, "Comment#3", 9));
    expectedOutput.add(insertRecord(10, 4L, "Comment#4", 10));
    expectedOutput.add(new Watermark(4L));
    // do a snapshot, data could be recovered from state
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    testHarness.close();
    expectedOutput.clear();
    operator = createSortOperator(inputRowType, rowTimeIdx, gComparator);
    testHarness = createTestHarness(operator);
    testHarness.initializeState(snapshot);
    testHarness.open();
    // late data will be dropped
    testHarness.processElement(insertRecord(5, 3L, "I am fine.", 6));
    testHarness.processWatermark(new Watermark(5L));
    expectedOutput.add(insertRecord(4, 5L, "Hello, how are you?", 4));
    expectedOutput.add(new Watermark(5L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    // those watermark has no effect
    testHarness.processWatermark(new Watermark(11L));
    testHarness.processWatermark(new Watermark(12L));
    expectedOutput.add(new Watermark(11L));
    expectedOutput.add(new Watermark(12L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}

Example 80 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class InputConversionOperatorTest method testWatermarkSuppression.

@Test
public void testWatermarkSuppression() throws Exception {
    final InputConversionOperator<Row> operator = new InputConversionOperator<>(createConverter(DataTypes.ROW(DataTypes.FIELD("f", DataTypes.INT()))), false, false, false, true);
    // would throw an exception otherwise because an output is not set
    operator.processWatermark(new Watermark(1000));
}