Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 71 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoStreamMapTest method testCoMap.

@Test
public void testCoMap() throws Exception {
    CoStreamMap<Double, Integer, String> operator = new CoStreamMap<Double, Integer, String>(new MyCoMap());
    TwoInputStreamOperatorTestHarness<Double, Integer, String> testHarness = new TwoInputStreamOperatorTestHarness<Double, Integer, String>(operator);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    testHarness.open();
    testHarness.processElement1(new StreamRecord<Double>(1.1d, initialTime + 1));
    testHarness.processElement1(new StreamRecord<Double>(1.2d, initialTime + 2));
    testHarness.processElement1(new StreamRecord<Double>(1.3d, initialTime + 3));
    testHarness.processWatermark1(new Watermark(initialTime + 3));
    testHarness.processElement1(new StreamRecord<Double>(1.4d, initialTime + 4));
    testHarness.processElement1(new StreamRecord<Double>(1.5d, initialTime + 5));
    testHarness.processElement2(new StreamRecord<Integer>(1, initialTime + 1));
    testHarness.processElement2(new StreamRecord<Integer>(2, initialTime + 2));
    testHarness.processWatermark2(new Watermark(initialTime + 2));
    testHarness.processElement2(new StreamRecord<Integer>(3, initialTime + 3));
    testHarness.processElement2(new StreamRecord<Integer>(4, initialTime + 4));
    testHarness.processElement2(new StreamRecord<Integer>(5, initialTime + 5));
    expectedOutput.add(new StreamRecord<String>("1.1", initialTime + 1));
    expectedOutput.add(new StreamRecord<String>("1.2", initialTime + 2));
    expectedOutput.add(new StreamRecord<String>("1.3", initialTime + 3));
    expectedOutput.add(new StreamRecord<String>("1.4", initialTime + 4));
    expectedOutput.add(new StreamRecord<String>("1.5", initialTime + 5));
    expectedOutput.add(new StreamRecord<String>("1", initialTime + 1));
    expectedOutput.add(new StreamRecord<String>("2", initialTime + 2));
    expectedOutput.add(new Watermark(initialTime + 2));
    expectedOutput.add(new StreamRecord<String>("3", initialTime + 3));
    expectedOutput.add(new StreamRecord<String>("4", initialTime + 4));
    expectedOutput.add(new StreamRecord<String>("5", initialTime + 5));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 72 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class KeyedCoProcessOperatorTest method testEventTimeTimerWithState.

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {
    KeyedCoProcessOperator<String, Integer, String, String> operator = new KeyedCoProcessOperator<>(new EventTimeTriggeringStatefulProcessFunction());
    TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>(operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark1(new Watermark(1));
    testHarness.processWatermark2(new Watermark(1));
    // should set timer for 6
    testHarness.processElement1(new StreamRecord<>(17, 0L));
    // should set timer for 6
    testHarness.processElement1(new StreamRecord<>(13, 0L));
    testHarness.processWatermark1(new Watermark(2));
    testHarness.processWatermark2(new Watermark(2));
    // should delete timer
    testHarness.processElement1(new StreamRecord<>(13, 1L));
    // should set timer for 7
    testHarness.processElement2(new StreamRecord<>("42", 1L));
    testHarness.processWatermark1(new Watermark(6));
    testHarness.processWatermark2(new Watermark(6));
    testHarness.processWatermark1(new Watermark(7));
    testHarness.processWatermark2(new Watermark(7));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(1L));
    expectedOutput.add(new StreamRecord<>("INPUT1:17", 0L));
    expectedOutput.add(new StreamRecord<>("INPUT1:13", 0L));
    expectedOutput.add(new Watermark(2L));
    expectedOutput.add(new StreamRecord<>("INPUT2:42", 1L));
    expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
    expectedOutput.add(new Watermark(6L));
    expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
    expectedOutput.add(new Watermark(7L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 73 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class RowTimeRangeBoundedPrecedingFunctionTest method testStateCleanup.

@Test
public void testStateCleanup() throws Exception {
    RowTimeRangeBoundedPrecedingFunction<RowData> function = new RowTimeRangeBoundedPrecedingFunction<>(aggsHandleFunction, accTypes, inputFieldTypes, 2000, 2);
    KeyedProcessOperator<RowData, RowData, RowData> operator = new KeyedProcessOperator<>(function);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    AbstractKeyedStateBackend stateBackend = (AbstractKeyedStateBackend) operator.getKeyedStateBackend();
    assertEquals("Initial state is not empty", 0, stateBackend.numKeyValueStateEntries());
    // put some records
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 500L));
    testHarness.processWatermark(new Watermark(1000L));
    // at this moment we expect the function to have some records in state
    testHarness.processWatermark(new Watermark(4000L));
    // at this moment the function should have cleaned up states
    assertEquals("State has not been cleaned up", 0, stateBackend.numKeyValueStateEntries());
}

Example 74 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class RowTimeRowsBoundedPrecedingFunctionTest method testLateRecordMetrics.

@Test
public void testLateRecordMetrics() throws Exception {
    RowTimeRowsBoundedPrecedingFunction<RowData> function = new RowTimeRowsBoundedPrecedingFunction<>(1000, 2000, aggsHandleFunction, accTypes, inputFieldTypes, 2000, 2);
    KeyedProcessOperator<RowData, RowData, RowData> operator = new KeyedProcessOperator<>(function);
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.open();
    Counter counter = function.getCounter();
    // put some records
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 100L));
    testHarness.processElement(insertRecord("key", 1L, 500L));
    testHarness.processWatermark(new Watermark(500L));
    // late record
    testHarness.processElement(insertRecord("key", 1L, 400L));
    assertEquals(1L, counter.getCount());
}

Example 75 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class WindowRankOperatorTest method testTop2WindowsWithoutRankNumber.

@Test
public void testTop2WindowsWithoutRankNumber() throws Exception {
    SlicingWindowOperator<RowData, ?> operator = WindowRankOperatorBuilder.builder().inputSerializer(INPUT_ROW_SER).shiftTimeZone(shiftTimeZone).keySerializer(KEY_SER).sortKeyComparator(GENERATED_SORT_KEY_COMPARATOR).sortKeySelector(SORT_KEY_SELECTOR).outputRankNumber(false).rankStart(1).rankEnd(2).windowEndIndex(WINDOW_END_INDEX).build();
    OneInputStreamOperatorTestHarness<RowData, RowData> testHarness = createTestHarness(operator);
    testHarness.setup(OUT_SERIALIZER_WITHOUT_RANK_NUMBER);
    testHarness.open();
    // process elements
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    // add elements out-of-order
    testHarness.processElement(insertRecord("key2", 1, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key2", 4, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key2", 5, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key2", 3, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key2", 2, toUtcTimestampMills(1999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key2", 7, toUtcTimestampMills(3999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key2", 8, toUtcTimestampMills(3999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key2", 1, toUtcTimestampMills(3999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key1", 2, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key1", 1, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key1", 3, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key1", 3, toUtcTimestampMills(999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key1", 4, toUtcTimestampMills(1999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key1", 6, toUtcTimestampMills(1999L, shiftTimeZone)));
    testHarness.processElement(insertRecord("key1", 7, toUtcTimestampMills(1999L, shiftTimeZone)));
    testHarness.processWatermark(new Watermark(999));
    expectedOutput.add(insertRecord("key1", 1, toUtcTimestampMills(999L, shiftTimeZone)));
    expectedOutput.add(insertRecord("key1", 2, toUtcTimestampMills(999L, shiftTimeZone)));
    expectedOutput.add(insertRecord("key2", 1, toUtcTimestampMills(999L, shiftTimeZone)));
    expectedOutput.add(insertRecord("key2", 3, toUtcTimestampMills(999L, shiftTimeZone)));
    expectedOutput.add(new Watermark(999));
    ASSERTER_WITHOUT_RANK_NUMBER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processWatermark(new Watermark(1999));
    expectedOutput.add(insertRecord("key1", 4, toUtcTimestampMills(1999L, shiftTimeZone)));
    expectedOutput.add(insertRecord("key1", 6, toUtcTimestampMills(1999L, shiftTimeZone)));
    expectedOutput.add(insertRecord("key2", 2, toUtcTimestampMills(1999L, shiftTimeZone)));
    expectedOutput.add(new Watermark(1999));
    ASSERTER_WITHOUT_RANK_NUMBER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    // do a snapshot, close and restore again
    testHarness.prepareSnapshotPreBarrier(0L);
    OperatorSubtaskState snapshot = testHarness.snapshot(0L, 0);
    testHarness.close();
    expectedOutput.clear();
    testHarness = createTestHarness(operator);
    testHarness.setup(OUT_SERIALIZER_WITHOUT_RANK_NUMBER);
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.processWatermark(new Watermark(3999));
    expectedOutput.add(insertRecord("key2", 1, toUtcTimestampMills(3999L, shiftTimeZone)));
    expectedOutput.add(insertRecord("key2", 7, toUtcTimestampMills(3999L, shiftTimeZone)));
    expectedOutput.add(new Watermark(3999));
    ASSERTER_WITHOUT_RANK_NUMBER.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}