Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 56 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class LegacyKeyedCoProcessOperatorTest method testEventTimeTimers.

@Test
public void testEventTimeTimers() throws Exception {
    LegacyKeyedCoProcessOperator<String, Integer, String, String> operator = new LegacyKeyedCoProcessOperator<>(new EventTimeTriggeringProcessFunction());
    TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>(operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processElement1(new StreamRecord<>(17, 42L));
    testHarness.processElement2(new StreamRecord<>("18", 42L));
    testHarness.processWatermark1(new Watermark(5));
    testHarness.processWatermark2(new Watermark(5));
    testHarness.processWatermark1(new Watermark(6));
    testHarness.processWatermark2(new Watermark(6));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>("INPUT1:17", 42L));
    expectedOutput.add(new StreamRecord<>("INPUT2:18", 42L));
    expectedOutput.add(new StreamRecord<>("1777", 5L));
    expectedOutput.add(new Watermark(5L));
    expectedOutput.add(new StreamRecord<>("1777", 6L));
    expectedOutput.add(new Watermark(6L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 57 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class KeyedProcessOperatorTest method testTimestampAndWatermarkQuerying.

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {
    KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME));
    OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark(new Watermark(17));
    testHarness.processElement(new StreamRecord<>(5, 12L));
    testHarness.processWatermark(new Watermark(42));
    testHarness.processElement(new StreamRecord<>(6, 13L));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(17L));
    expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L));
    expectedOutput.add(new Watermark(42L));
    expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 58 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class KeyedProcessOperatorTest method testSnapshotAndRestore.

@Test
public void testSnapshotAndRestore() throws Exception {
    final int expectedKey = 5;
    KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey));
    OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(expectedKey, 12L));
    // snapshot and restore from scratch
    OperatorSubtaskState snapshot = testHarness.snapshot(0, 0);
    testHarness.close();
    operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey));
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setup();
    testHarness.initializeState(snapshot);
    testHarness.open();
    testHarness.setProcessingTime(5);
    testHarness.processWatermark(new Watermark(6));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>("PROC:1777"));
    expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L));
    expectedOutput.add(new Watermark(6));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 59 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class KeyedProcessOperatorTest method testEventTimeTimerWithState.

/**
 * Verifies that we don't have leakage between different keys.
 */
@Test
public void testEventTimeTimerWithState() throws Exception {
    KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME));
    OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark(new Watermark(1));
    // should set timer for 6
    testHarness.processElement(new StreamRecord<>(17, 0L));
    // should set timer for 6
    testHarness.processElement(new StreamRecord<>(13, 0L));
    testHarness.processWatermark(new Watermark(2));
    // should set timer for 7
    testHarness.processElement(new StreamRecord<>(42, 1L));
    // should delete timer
    testHarness.processElement(new StreamRecord<>(13, 1L));
    testHarness.processWatermark(new Watermark(6));
    testHarness.processWatermark(new Watermark(7));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(1L));
    expectedOutput.add(new StreamRecord<>("INPUT:17", 0L));
    expectedOutput.add(new StreamRecord<>("INPUT:13", 0L));
    expectedOutput.add(new Watermark(2L));
    expectedOutput.add(new StreamRecord<>("INPUT:42", 1L));
    expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
    expectedOutput.add(new Watermark(6L));
    expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
    expectedOutput.add(new Watermark(7L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 60 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class UnorderedStreamElementQueueTest method testCompletionOrder.

/**
 * Tests that only elements before the oldest watermark are returned if they are completed.
 */
@Test
public void testCompletionOrder() {
    final UnorderedStreamElementQueue<Integer> queue = new UnorderedStreamElementQueue<>(8);
    ResultFuture<Integer> record1 = putSuccessfully(queue, new StreamRecord<>(1, 0L));
    ResultFuture<Integer> record2 = putSuccessfully(queue, new StreamRecord<>(2, 1L));
    putSuccessfully(queue, new Watermark(2L));
    ResultFuture<Integer> record3 = putSuccessfully(queue, new StreamRecord<>(3, 3L));
    ResultFuture<Integer> record4 = putSuccessfully(queue, new StreamRecord<>(4, 4L));
    putSuccessfully(queue, new Watermark(5L));
    ResultFuture<Integer> record5 = putSuccessfully(queue, new StreamRecord<>(5, 6L));
    ResultFuture<Integer> record6 = putSuccessfully(queue, new StreamRecord<>(6, 7L));
    Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
    Assert.assertEquals(8, queue.size());
    Assert.assertFalse(queue.isEmpty());
    // this should not make any item completed, because R3 is behind W1
    record3.complete(Arrays.asList(13));
    Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
    Assert.assertEquals(8, queue.size());
    Assert.assertFalse(queue.isEmpty());
    record2.complete(Arrays.asList(12));
    Assert.assertEquals(Arrays.asList(new StreamRecord<>(12, 1L)), popCompleted(queue));
    Assert.assertEquals(7, queue.size());
    Assert.assertFalse(queue.isEmpty());
    // Should not be completed because R1 has not been completed yet
    record6.complete(Arrays.asList(16));
    record4.complete(Arrays.asList(14));
    Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
    Assert.assertEquals(7, queue.size());
    Assert.assertFalse(queue.isEmpty());
    // Now W1, R3, R4 and W2 are completed and should be pollable
    record1.complete(Arrays.asList(11));
    Assert.assertEquals(Arrays.asList(new StreamRecord<>(11, 0L), new Watermark(2L), new StreamRecord<>(13, 3L), new StreamRecord<>(14, 4L), new Watermark(5L), new StreamRecord<>(16, 7L)), popCompleted(queue));
    Assert.assertEquals(1, queue.size());
    Assert.assertFalse(queue.isEmpty());
    // only R5 left in the queue
    record5.complete(Arrays.asList(15));
    Assert.assertEquals(Arrays.asList(new StreamRecord<>(15, 6L)), popCompleted(queue));
    Assert.assertEquals(0, queue.size());
    Assert.assertTrue(queue.isEmpty());
    Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
}