Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 66 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamSinkOperatorTest method testTimeQuerying.

/**
 * Verify that we can correctly query watermark, processing time and the timestamp from the
 * context.
 */
@Test
public void testTimeQuerying() throws Exception {
    BufferingQueryingSink<String> bufferingSink = new BufferingQueryingSink<>();
    StreamSink<String> operator = new StreamSink<>(bufferingSink);
    OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark(new Watermark(17));
    testHarness.setProcessingTime(12);
    testHarness.processElement(new StreamRecord<>("Hello", 12L));
    testHarness.processWatermark(new Watermark(42));
    testHarness.setProcessingTime(15);
    testHarness.processElement(new StreamRecord<>("Ciao", 13L));
    testHarness.processWatermark(new Watermark(42));
    testHarness.setProcessingTime(15);
    testHarness.processElement(new StreamRecord<>("Ciao"));
    assertThat(bufferingSink.data.size(), is(3));
    assertThat(bufferingSink.data, contains(new Tuple4<>(17L, 12L, 12L, "Hello"), new Tuple4<>(42L, 15L, 13L, "Ciao"), new Tuple4<>(42L, 15L, null, "Ciao")));
    assertThat(bufferingSink.watermarks.size(), is(3));
    assertThat(bufferingSink.watermarks, contains(new org.apache.flink.api.common.eventtime.Watermark(17L), new org.apache.flink.api.common.eventtime.Watermark(42L), new org.apache.flink.api.common.eventtime.Watermark(42L)));
    testHarness.close();
}

Example 67 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamElementQueueTest method testWatermarkOnly.

/**
 * Tests two adjacent watermarks can be processed successfully.
 */
@Test
public void testWatermarkOnly() {
    final StreamElementQueue<Integer> queue = createStreamElementQueue(2);
    putSuccessfully(queue, new Watermark(2L));
    putSuccessfully(queue, new Watermark(5L));
    Assert.assertEquals(2, queue.size());
    Assert.assertFalse(queue.isEmpty());
    Assert.assertEquals(Arrays.asList(new Watermark(2L), new Watermark(5L)), popCompleted(queue));
    Assert.assertEquals(0, queue.size());
    Assert.assertTrue(queue.isEmpty());
    Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
}

Example 68 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamElementQueueTest method testPut.

@Test
public void testPut() {
    StreamElementQueue<Integer> queue = createStreamElementQueue(2);
    Watermark watermark = new Watermark(0L);
    StreamRecord<Integer> streamRecord = new StreamRecord<>(42, 1L);
    // add two elements to reach capacity
    assertTrue(queue.tryPut(watermark).isPresent());
    assertTrue(queue.tryPut(streamRecord).isPresent());
    assertEquals(2, queue.size());
    // queue full, cannot add new element
    assertFalse(queue.tryPut(new Watermark(2L)).isPresent());
    // check if expected values are returned (for checkpointing)
    assertEquals(Arrays.asList(watermark, streamRecord), queue.values());
}

Example 69 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoBroadcastWithKeyedOperatorTest method testNoKeyedStateOnBroadcastSide.

@Test
public void testNoKeyedStateOnBroadcastSide() throws Exception {
    boolean exceptionThrown = false;
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new KeyedBroadcastProcessFunction<String, String, Integer, String>() {

        private static final long serialVersionUID = -1725365436500098384L;

        private final ValueStateDescriptor<String> valueState = new ValueStateDescriptor<>("any", BasicTypeInfo.STRING_TYPE_INFO);

        @Override
        public void processBroadcastElement(Integer value, Context ctx, Collector<String> out) throws Exception {
            getRuntimeContext().getState(valueState).value();
        }

        @Override
        public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        // do nothing
        }
    })) {
        testHarness.processWatermark1(new Watermark(10L));
        testHarness.processWatermark2(new Watermark(10L));
        testHarness.processElement2(new StreamRecord<>(5, 12L));
    } catch (NullPointerException e) {
        assertEquals("No key set. This method should not be called outside of a keyed context.", e.getMessage());
        exceptionThrown = true;
    }
    if (!exceptionThrown) {
        fail("No exception thrown");
    }
}

Example 70 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoProcessOperatorTest method testTimestampAndWatermarkQuerying.

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {
    CoProcessOperator<Integer, String, String> operator = new CoProcessOperator<>(new WatermarkQueryingProcessFunction());
    TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new TwoInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark1(new Watermark(17));
    testHarness.processWatermark2(new Watermark(17));
    testHarness.processElement1(new StreamRecord<>(5, 12L));
    testHarness.processWatermark1(new Watermark(42));
    testHarness.processWatermark2(new Watermark(42));
    testHarness.processElement2(new StreamRecord<>("6", 13L));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(17L));
    expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L));
    expectedOutput.add(new Watermark(42L));
    expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}