Examples with RowData org.apache.flink.table.data.RowData used on opensource projects

Example 81 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class StreamArrowPythonGroupWindowAggregateFunctionOperatorTest method testFinishBundleTriggeredOnCheckpoint.

@Test
public void testFinishBundleTriggeredOnCheckpoint() throws Exception {
    Configuration conf = new Configuration();
    conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(conf);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c2", 0L, 0L), initialTime + 1));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c4", 1L, 6000L), initialTime + 2));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c6", 2L, 10000L), initialTime + 3));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c2", "c8", 3L, 0L), initialTime + 4));
    testHarness.processWatermark(new Watermark(10000L));
    // checkpoint trigger finishBundle
    testHarness.prepareSnapshotPreBarrier(0L);
    expectedOutput.add(new Watermark(10000L));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", 0L, TimestampData.fromEpochMillis(-5000L), TimestampData.fromEpochMillis(5000L))));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", 3L, TimestampData.fromEpochMillis(-5000L), TimestampData.fromEpochMillis(5000L))));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", 3L, TimestampData.fromEpochMillis(0L), TimestampData.fromEpochMillis(10000L))));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", 0L, TimestampData.fromEpochMillis(0L), TimestampData.fromEpochMillis(10000L))));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processWatermark(20000L);
    testHarness.close();
    expectedOutput.add(new Watermark(20000L));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", 1L, TimestampData.fromEpochMillis(5000L), TimestampData.fromEpochMillis(15000L))));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", 2L, TimestampData.fromEpochMillis(10000L), TimestampData.fromEpochMillis(20000L))));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 82 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class PythonStreamGroupAggregateOperatorTest method testStateCleanupTimer.

@Test
public void testStateCleanupTimer() throws Exception {
    Configuration conf = new Configuration();
    conf.setString("table.exec.state.ttl", "100");
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(conf);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.setProcessingTime(0L);
    testHarness.processElement(new StreamRecord<>(newRow(true, "c1", 0L), initialTime + 1));
    testHarness.setProcessingTime(500L);
    testHarness.processElement(new StreamRecord<>(newRow(true, "c2", 1L), initialTime + 2));
    testHarness.setProcessingTime(599L);
    testHarness.processElement(new StreamRecord<>(newRow(true, "c2", 2L), initialTime + 3));
    testHarness.setProcessingTime(1000L);
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "state_cleanup_triggered: c1", 100L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", 1L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", 2L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "state_cleanup_triggered: c2", 699L)));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 83 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class StreamArrowPythonRowTimeBoundedRangeOperatorTest method testFinishBundleTriggeredByCount.

@Test
public void testFinishBundleTriggeredByCount() throws Exception {
    Configuration conf = new Configuration();
    conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 4);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(conf);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c2", 0L, 1L), initialTime + 1));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c4", 1L, 1L), initialTime + 2));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c6", 2L, 10L), initialTime + 3));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c2", "c8", 3L, 2L), initialTime + 3));
    assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput());
    testHarness.processWatermark(new Watermark(1000L));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c4", 1L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", "c8", 3L, 2L, 3L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c6", 2L, 10L, 2L)));
    expectedOutput.add(new Watermark(1000L));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 84 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class StreamArrowPythonRowTimeBoundedRangeOperatorTest method testOverWindowAggregateFunction.

@Test
public void testOverWindowAggregateFunction() throws Exception {
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(new Configuration());
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c2", 0L, 1L), initialTime + 1));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c4", 1L, 1L), initialTime + 2));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c6", 2L, 10L), initialTime + 3));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c2", "c8", 3L, 2L), initialTime + 3));
    testHarness.processWatermark(Long.MAX_VALUE);
    testHarness.close();
    expectedOutput.add(new Watermark(Long.MAX_VALUE));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c4", 1L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", "c8", 3L, 2L, 3L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c6", 2L, 10L, 2L)));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 85 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class StreamArrowPythonRowTimeBoundedRangeOperatorTest method testFinishBundleTriggeredByTime.

@Test
public void testFinishBundleTriggeredByTime() throws Exception {
    Configuration conf = new Configuration();
    conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10);
    conf.setLong(PythonOptions.MAX_BUNDLE_TIME_MILLS, 1000L);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(conf);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c2", 0L, 1L), initialTime + 1));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c4", 1L, 1L), initialTime + 2));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c6", 2L, 10L), initialTime + 3));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c2", "c8", 3L, 2L), initialTime + 3));
    testHarness.processWatermark(new Watermark(10000L));
    expectedOutput.add(new Watermark(10000L));
    assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput());
    testHarness.setProcessingTime(1000L);
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c4", 1L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", "c8", 3L, 2L, 3L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c6", 2L, 10L, 2L)));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}