Examples with RowDataHarnessAssertor org.apache.flink.table.runtime.util.RowDataHarnessAssertor used on opensource projects

Example 1 with RowDataHarnessAssertor

use of org.apache.flink.table.runtime.util.RowDataHarnessAssertor in project flink by apache.

the class RowTimeSortOperatorTest method testSortOnTwoFields.

@Test
public void testSortOnTwoFields() throws Exception {
    InternalTypeInfo<RowData> inputRowType = InternalTypeInfo.ofFields(new IntType(), new BigIntType(), VarCharType.STRING_TYPE, new IntType());
    // Note: RowTimeIdx must be 0 in product environment, the value is 1 here just for simplify
    // the testing
    int rowTimeIdx = 1;
    GeneratedRecordComparator gComparator = new GeneratedRecordComparator("", "", new Object[0]) {

        private static final long serialVersionUID = -6067266199060901331L;

        @Override
        public RecordComparator newInstance(ClassLoader classLoader) {
            return IntRecordComparator.INSTANCE;
        }
    };
    RowDataHarnessAssertor assertor = new RowDataHarnessAssertor(inputRowType.toRowFieldTypes());
    RowTimeSortOperator operator = createSortOperator(inputRowType, rowTimeIdx, gComparator);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    testHarness.processElement(insertRecord(3, 3L, "Hello world", 3));
    testHarness.processElement(insertRecord(2, 2L, "Hello", 2));
    testHarness.processElement(insertRecord(6, 2L, "Luke Skywalker", 6));
    testHarness.processElement(insertRecord(5, 3L, "I am fine.", 5));
    testHarness.processElement(insertRecord(7, 1L, "Comment#1", 7));
    testHarness.processElement(insertRecord(9, 4L, "Comment#3", 9));
    testHarness.processElement(insertRecord(10, 4L, "Comment#4", 10));
    testHarness.processElement(insertRecord(8, 4L, "Comment#2", 8));
    testHarness.processElement(insertRecord(1, 1L, "Hi", 2));
    testHarness.processElement(insertRecord(1, 1L, "Hi", 1));
    testHarness.processElement(insertRecord(4, 3L, "Helloworld, how are you?", 4));
    testHarness.processElement(insertRecord(4, 5L, "Hello, how are you?", 4));
    testHarness.processWatermark(new Watermark(4L));
    List<Object> expectedOutput = new ArrayList<>();
    expectedOutput.add(insertRecord(1, 1L, "Hi", 2));
    expectedOutput.add(insertRecord(1, 1L, "Hi", 1));
    expectedOutput.add(insertRecord(7, 1L, "Comment#1", 7));
    expectedOutput.add(insertRecord(2, 2L, "Hello", 2));
    expectedOutput.add(insertRecord(6, 2L, "Luke Skywalker", 6));
    expectedOutput.add(insertRecord(3, 3L, "Hello world", 3));
    expectedOutput.add(insertRecord(4, 3L, "Helloworld, how are you?", 4));
    expectedOutput.add(insertRecord(5, 3L, "I am fine.", 5));
    expectedOutput.add(insertRecord(8, 4L, "Comment#2", 8));
    expectedOutput.add(insertRecord(9, 4L, "Comment#3", 9));
    expectedOutput.add(insertRecord(10, 4L, "Comment#4", 10));
    expectedOutput.add(new Watermark(4L));
    // do a snapshot, data could be recovered from state
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    testHarness.close();
    expectedOutput.clear();
    operator = createSortOperator(inputRowType, rowTimeIdx, gComparator);
    testHarness = createTestHarness(operator);
    testHarness.initializeState(snapshot);
    testHarness.open();
    // late data will be dropped
    testHarness.processElement(insertRecord(5, 3L, "I am fine.", 6));
    testHarness.processWatermark(new Watermark(5L));
    expectedOutput.add(insertRecord(4, 5L, "Hello, how are you?", 4));
    expectedOutput.add(new Watermark(5L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    // those watermark has no effect
    testHarness.processWatermark(new Watermark(11L));
    testHarness.processWatermark(new Watermark(12L));
    expectedOutput.add(new Watermark(11L));
    expectedOutput.add(new Watermark(12L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}

Example 2 with RowDataHarnessAssertor

use of org.apache.flink.table.runtime.util.RowDataHarnessAssertor in project flink by apache.

the class WindowOperatorTest method testProcessingTimeSessionWindows.

@Test
public void testProcessingTimeSessionWindows() throws Throwable {
    closeCalled.set(0);
    WindowOperator operator = WindowOperatorBuilder.builder().withInputFields(inputFieldTypes).withShiftTimezone(shiftTimeZone).session(Duration.ofSeconds(3)).withProcessingTime().aggregateAndBuild(getTimeWindowAggFunction(), equaliser, accTypes, aggResultTypes, windowTypes);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    RowDataHarnessAssertor assertor = new RowDataHarnessAssertor(outputType.toRowFieldTypes(), new GenericRowRecordSortComparator(0, VarCharType.STRING_TYPE));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    // timestamp is ignored in processing time
    testHarness.setProcessingTime(3);
    testHarness.processElement(insertRecord("key2", 1, 1L));
    testHarness.setProcessingTime(1000);
    testHarness.processElement(insertRecord("key2", 1, 1002L));
    testHarness.setProcessingTime(5000);
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, localMills(3L), localMills(4000L), localMills(3999L))));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(insertRecord("key2", 1, 5000L));
    testHarness.processElement(insertRecord("key2", 1, 5000L));
    testHarness.processElement(insertRecord("key1", 1, 5000L));
    testHarness.processElement(insertRecord("key1", 1, 5000L));
    testHarness.processElement(insertRecord("key1", 1, 5000L));
    testHarness.setProcessingTime(10000);
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key2", 2L, 2L, localMills(5000L), localMills(8000L), localMills(7999L))));
    expectedOutput.addAll(doubleRecord(isTableAggregate, insertRecord("key1", 3L, 3L, localMills(5000L), localMills(8000L), localMills(7999L))));
    assertor.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 3 with RowDataHarnessAssertor

use of org.apache.flink.table.runtime.util.RowDataHarnessAssertor in project flink by apache.

the class RowTimeSortOperatorTest method testOnlySortOnRowTime.

@Test
public void testOnlySortOnRowTime() throws Exception {
    InternalTypeInfo<RowData> inputRowType = InternalTypeInfo.ofFields(new BigIntType(), new BigIntType(), VarCharType.STRING_TYPE, new IntType());
    int rowTimeIdx = 0;
    RowDataHarnessAssertor assertor = new RowDataHarnessAssertor(inputRowType.toRowFieldTypes());
    RowTimeSortOperator operator = createSortOperator(inputRowType, rowTimeIdx, null);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    testHarness.processElement(insertRecord(3L, 2L, "Hello world", 3));
    testHarness.processElement(insertRecord(2L, 2L, "Hello", 2));
    testHarness.processElement(insertRecord(6L, 3L, "Luke Skywalker", 6));
    testHarness.processElement(insertRecord(5L, 3L, "I am fine.", 5));
    testHarness.processElement(insertRecord(7L, 4L, "Comment#1", 7));
    testHarness.processElement(insertRecord(9L, 4L, "Comment#3", 9));
    testHarness.processElement(insertRecord(10L, 4L, "Comment#4", 10));
    testHarness.processElement(insertRecord(8L, 4L, "Comment#2", 8));
    testHarness.processElement(insertRecord(1L, 1L, "Hi", 2));
    testHarness.processElement(insertRecord(1L, 1L, "Hi", 1));
    testHarness.processElement(insertRecord(4L, 3L, "Helloworld, how are you?", 4));
    testHarness.processWatermark(new Watermark(9L));
    List<Object> expectedOutput = new ArrayList<>();
    expectedOutput.add(insertRecord(1L, 1L, "Hi", 2));
    expectedOutput.add(insertRecord(1L, 1L, "Hi", 1));
    expectedOutput.add(insertRecord(2L, 2L, "Hello", 2));
    expectedOutput.add(insertRecord(3L, 2L, "Hello world", 3));
    expectedOutput.add(insertRecord(4L, 3L, "Helloworld, how are you?", 4));
    expectedOutput.add(insertRecord(5L, 3L, "I am fine.", 5));
    expectedOutput.add(insertRecord(6L, 3L, "Luke Skywalker", 6));
    expectedOutput.add(insertRecord(7L, 4L, "Comment#1", 7));
    expectedOutput.add(insertRecord(8L, 4L, "Comment#2", 8));
    expectedOutput.add(insertRecord(9L, 4L, "Comment#3", 9));
    expectedOutput.add(new Watermark(9L));
    // do a snapshot, data could be recovered from state
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    testHarness.close();
    expectedOutput.clear();
    operator = createSortOperator(inputRowType, rowTimeIdx, null);
    testHarness = createTestHarness(operator);
    testHarness.initializeState(snapshot);
    testHarness.open();
    // late data will be dropped
    testHarness.processElement(insertRecord(5L, 3L, "I am fine.", 6));
    testHarness.processWatermark(new Watermark(10L));
    expectedOutput.add(insertRecord(10L, 4L, "Comment#4", 10));
    expectedOutput.add(new Watermark(10L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
    // those watermark has no effect
    testHarness.processWatermark(new Watermark(11L));
    testHarness.processWatermark(new Watermark(12L));
    expectedOutput.add(new Watermark(11L));
    expectedOutput.add(new Watermark(12L));
    assertor.assertOutputEquals("output wrong.", expectedOutput, testHarness.getOutput());
}