Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 61 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoBroadcastWithNonKeyedOperatorTest method testNoKeyedStateOnNonBroadcastSide.

@Test
public void testNoKeyedStateOnNonBroadcastSide() throws Exception {
    boolean exceptionThrown = false;
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(new BroadcastProcessFunction<String, Integer, String>() {

        private static final long serialVersionUID = -1725365436500098384L;

        private final ValueStateDescriptor<String> valueState = new ValueStateDescriptor<>("any", BasicTypeInfo.STRING_TYPE_INFO);

        @Override
        public void processBroadcastElement(Integer value, Context ctx, Collector<String> out) throws Exception {
        // do nothing
        }

        @Override
        public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
            getRuntimeContext().getState(valueState).value();
        }
    })) {
        testHarness.processWatermark1(new Watermark(10L));
        testHarness.processWatermark2(new Watermark(10L));
        testHarness.processElement1(new StreamRecord<>("5", 12L));
    } catch (NullPointerException e) {
        Assert.assertEquals("Keyed state can only be used on a 'keyed stream', i.e., after a 'keyBy()' operation.", e.getMessage());
        exceptionThrown = true;
    }
    if (!exceptionThrown) {
        Assert.fail("No exception thrown");
    }
}

Example 62 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoBroadcastWithNonKeyedOperatorTest method testBroadcastState.

@Test
public void testBroadcastState() throws Exception {
    final Set<String> keysToRegister = new HashSet<>();
    keysToRegister.add("test1");
    keysToRegister.add("test2");
    keysToRegister.add("test3");
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(new TestFunction(keysToRegister), STATE_DESCRIPTOR)) {
        testHarness.processWatermark1(new Watermark(10L));
        testHarness.processWatermark2(new Watermark(10L));
        testHarness.processElement2(new StreamRecord<>(5, 12L));
        testHarness.processWatermark1(new Watermark(40L));
        testHarness.processWatermark2(new Watermark(40L));
        testHarness.processElement1(new StreamRecord<>("6", 13L));
        testHarness.processElement1(new StreamRecord<>("6", 15L));
        testHarness.processWatermark1(new Watermark(50L));
        testHarness.processWatermark2(new Watermark(50L));
        Queue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
        expectedOutput.add(new Watermark(10L));
        expectedOutput.add(new StreamRecord<>("5WM:10 TS:12", 12L));
        expectedOutput.add(new Watermark(40L));
        expectedOutput.add(new StreamRecord<>("6WM:40 TS:13", 13L));
        expectedOutput.add(new StreamRecord<>("6WM:40 TS:15", 15L));
        expectedOutput.add(new Watermark(50L));
        TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    }
}

Example 63 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoBroadcastWithNonKeyedOperatorTest method testNoKeyedStateOnBroadcastSide.

@Test
public void testNoKeyedStateOnBroadcastSide() throws Exception {
    boolean exceptionThrown = false;
    try (TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(new BroadcastProcessFunction<String, Integer, String>() {

        private static final long serialVersionUID = -1725365436500098384L;

        private final ValueStateDescriptor<String> valueState = new ValueStateDescriptor<>("any", BasicTypeInfo.STRING_TYPE_INFO);

        @Override
        public void processBroadcastElement(Integer value, Context ctx, Collector<String> out) throws Exception {
            getRuntimeContext().getState(valueState).value();
        }

        @Override
        public void processElement(String value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        // do nothing
        }
    })) {
        testHarness.processWatermark1(new Watermark(10L));
        testHarness.processWatermark2(new Watermark(10L));
        testHarness.processElement2(new StreamRecord<>(5, 12L));
    } catch (NullPointerException e) {
        Assert.assertEquals("Keyed state can only be used on a 'keyed stream', i.e., after a 'keyBy()' operation.", e.getMessage());
        exceptionThrown = true;
    }
    if (!exceptionThrown) {
        Assert.fail("No exception thrown");
    }
}

Example 64 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamSourceContextIdleDetectionTests method testAutomaticWatermarkContext.

/**
 * Test scenario (idleTimeout = 100, watermarkInterval = 40): (1) Start from 20 as initial time.
 * (2) As soon as time reaches 120, status should have been toggled to IDLE. (3) After some
 * arbitrary time (until 320), the status should remain IDLE, and no watermarks should have been
 * emitted. (4) Emit a record at 330. Status should become ACTIVE. This should schedule a
 * idleness detection to be fired at 430. (5) Emit another record at 350 (which is before the
 * next check). This should make the idleness check pass. (6) Advance time to 430 and trigger
 * idleness detection. The status should still be ACTIVE due to step (5). This should schedule a
 * idleness detection to be fired at 530. (7) Advance time to 460, in which a watermark emission
 * task should be fired. Idleness detection should have been "piggy-backed" in the task,
 * allowing the status to be toggled to IDLE before the next actual idle detection task at 530.
 *
 * <p>Inline comments will refer to the corresponding tested steps in the scenario.
 */
@Test
public void testAutomaticWatermarkContext() throws Exception {
    long watermarkInterval = 40;
    long idleTimeout = 100;
    long initialTime = 20;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(initialTime);
    final List<StreamElement> output = new ArrayList<>();
    final List<StreamElement> expectedOutput = new ArrayList<>();
    SourceFunction.SourceContext<String> context = StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime, processingTimeService, new Object(), new CollectorOutput<String>(output), watermarkInterval, idleTimeout, true);
    // -------------------------- begin test scenario --------------------------
    // corresponds to step (2) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + watermarkInterval);
    expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
    processingTimeService.setCurrentTime(initialTime + 2 * watermarkInterval);
    expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
    processingTimeService.setCurrentTime(initialTime + idleTimeout);
    expectedOutput.add(WatermarkStatus.IDLE);
    assertEquals(expectedOutput, output);
    // corresponds to step (3) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 4 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout);
    processingTimeService.setCurrentTime(initialTime + 6 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 7 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout);
    assertEquals(expectedOutput, output);
    // corresponds to step (4) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            expectedOutput.add(WatermarkStatus.ACTIVE);
            context.collect("msg");
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertEquals(expectedOutput, output);
            break;
        case COLLECT_WITH_TIMESTAMP:
            expectedOutput.add(WatermarkStatus.ACTIVE);
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            // for emitWatermark, since the watermark will be blocked,
            // it should not make the status become active;
            // from here on, the status should remain idle for the emitWatermark variant test
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            assertEquals(expectedOutput, output);
    }
    // corresponds to step (5) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 8 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + 3 * idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            context.collect("msg");
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            assertEquals(expectedOutput, output);
            break;
        case COLLECT_WITH_TIMESTAMP:
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            assertEquals(expectedOutput, output);
    }
    processingTimeService.setCurrentTime(initialTime + 9 * watermarkInterval);
    switch(testMethod) {
        case COLLECT:
        case COLLECT_WITH_TIMESTAMP:
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            assertEquals(expectedOutput, output);
    }
    processingTimeService.setCurrentTime(initialTime + 10 * watermarkInterval);
    switch(testMethod) {
        case COLLECT:
        case COLLECT_WITH_TIMESTAMP:
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            assertEquals(expectedOutput, output);
    }
    // corresponds to step (6) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 4 * idleTimeout + idleTimeout / 10);
    assertEquals(expectedOutput, output);
    // corresponds to step (7) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 11 * watermarkInterval);
    // emit watermark does not change the previous status
    if (testMethod != TestMethod.EMIT_WATERMARK) {
        expectedOutput.add(WatermarkStatus.IDLE);
    }
    assertEquals(expectedOutput, output);
}

Example 65 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class KeyedCoProcessOperatorTest method testTimestampAndWatermarkQuerying.

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {
    KeyedCoProcessOperator<String, Integer, String, String> operator = new KeyedCoProcessOperator<>(new WatermarkQueryingProcessFunction());
    TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>(operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark1(new Watermark(17));
    testHarness.processWatermark2(new Watermark(17));
    testHarness.processElement1(new StreamRecord<>(5, 12L));
    testHarness.processWatermark1(new Watermark(42));
    testHarness.processWatermark2(new Watermark(42));
    testHarness.processElement2(new StreamRecord<>("6", 13L));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(17L));
    expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L));
    expectedOutput.add(new Watermark(42L));
    expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}