Examples with RowData org.apache.flink.table.data.RowData used on opensource projects

Example 86 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class StreamArrowPythonRowTimeBoundedRowsOperatorTest method testFinishBundleTriggeredByTime.

@Test
public void testFinishBundleTriggeredByTime() throws Exception {
    Configuration conf = new Configuration();
    conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10);
    conf.setLong(PythonOptions.MAX_BUNDLE_TIME_MILLS, 1000L);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(conf);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c2", 0L, 1L), initialTime + 1));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c4", 1L, 1L), initialTime + 2));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c6", 2L, 10L), initialTime + 3));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c2", "c8", 3L, 2L), initialTime + 3));
    testHarness.processWatermark(new Watermark(10000L));
    expectedOutput.add(new Watermark(10000L));
    assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput());
    testHarness.setProcessingTime(1000L);
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c4", 1L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", "c8", 3L, 2L, 3L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c6", 2L, 10L, 1L)));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 87 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class StreamArrowPythonRowTimeBoundedRowsOperatorTest method testOverWindowAggregateFunction.

@Test
public void testOverWindowAggregateFunction() throws Exception {
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(new Configuration());
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c2", 0L, 1L), initialTime + 1));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c4", 1L, 1L), initialTime + 2));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c1", "c6", 2L, 10L), initialTime + 3));
    testHarness.processElement(new StreamRecord<>(newBinaryRow(true, "c2", "c8", 3L, 2L), initialTime + 3));
    testHarness.processWatermark(Long.MAX_VALUE);
    testHarness.close();
    expectedOutput.add(new Watermark(Long.MAX_VALUE));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c4", 1L, 1L, 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", "c8", 3L, 2L, 3L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c6", 2L, 10L, 1L)));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 88 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class PassThroughPythonStreamGroupWindowAggregateOperator method cleanWindowIfNeeded.

private void cleanWindowIfNeeded(RowData key, TimeWindow window, long currentTime) throws IOException {
    if (currentTime == cleanupTime(window)) {
        // 1. delete state
        // only output first value in group window.
        DataOutputSerializer output = new DataOutputSerializer(1);
        RowData windowProperty = windowExtractor.apply(window);
        windowAggResult.replace(GenericRowData.of(StringData.fromString("state_cleanup_triggered: " + key.getString(0).toString() + " : " + window)), GenericRowData.of(0L));
        reuseJoinedRow.replace(windowAggResult, windowProperty);
        reusePythonRowData.setField(1, reuseJoinedRow);
        udfOutputTypeSerializer.serialize(reusePythonRowData, output);
        resultBuffer.add(output.getCopyOfBuffer());
        // 2. delete window timer
        if (windowAssigner.isEventTime()) {
            deleteEventTimeTimer(key, window);
        } else {
            deleteProcessingTimeTimer(key, window);
        }
    }
}

Example 89 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class PassThroughPythonStreamGroupWindowAggregateOperator method triggerWindowProcess.

private void triggerWindowProcess(RowData key, TimeWindow window) throws Exception {
    DataOutputSerializer output = new DataOutputSerializer(1);
    Iterable<RowData> currentWindowAccumulateData = windowAccumulateData.get(key.getString(0).toString()).get(window);
    Iterable<RowData> currentWindowRetractData = windowRetractData.get(key.getString(0).toString()).get(window);
    if (currentWindowAccumulateData != null) {
        for (RowData accumulateData : currentWindowAccumulateData) {
            if (!hasRetractData(accumulateData, currentWindowRetractData)) {
                // only output first value in group window.
                RowData aggResult = aggExtracter.apply(accumulateData);
                RowData windowProperty = windowExtractor.apply(window);
                windowAggResult.replace(key, aggResult);
                reuseJoinedRow.replace(windowAggResult, windowProperty);
                reusePythonRowData.setField(1, reuseJoinedRow);
                udfOutputTypeSerializer.serialize(reusePythonRowData, output);
                resultBuffer.add(output.getCopyOfBuffer());
                break;
            }
        }
    }
}

Example 90 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class PythonStreamGroupTableAggregateOperatorTest method testFinishBundleTriggeredOnCheckpoint.

@Test
public void testFinishBundleTriggeredOnCheckpoint() throws Exception {
    Configuration conf = new Configuration();
    conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = getTestHarness(conf);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(newRow(true, "c1", 0L), initialTime + 1));
    testHarness.processElement(new StreamRecord<>(newRow(true, "c2", 1L), initialTime + 2));
    testHarness.processElement(new StreamRecord<>(newRow(true, "c3", 2L), initialTime + 3));
    // checkpoint trigger finishBundle
    testHarness.prepareSnapshotPreBarrier(0L);
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c1", 0L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", 1L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c2", 1L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c3", 2L)));
    expectedOutput.add(new StreamRecord<>(newRow(true, "c3", 2L)));
    assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}