Examples with Watermark org.apache.flink.streaming.api.watermark.Watermark used on opensource projects

Example 41 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoProcessOperatorTest method testTimestampAndWatermarkQuerying.

@Test
public void testTimestampAndWatermarkQuerying() throws Exception {
    CoProcessOperator<Integer, String, String> operator = new CoProcessOperator<>(new WatermarkQueryingProcessFunction());
    TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new TwoInputStreamOperatorTestHarness<>(operator);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark1(new Watermark(17));
    testHarness.processWatermark2(new Watermark(17));
    testHarness.processElement1(new StreamRecord<>(5, 12L));
    testHarness.processWatermark1(new Watermark(42));
    testHarness.processWatermark2(new Watermark(42));
    testHarness.processElement2(new StreamRecord<>("6", 13L));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(17L));
    expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L));
    expectedOutput.add(new Watermark(42L));
    expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}

Example 42 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamSourceContextIdleDetectionTests method testAutomaticWatermarkContext.

/**
	 * Test scenario (idleTimeout = 100, watermarkInterval = 40):
	 * (1) Start from 20 as initial time.
	 * (2) As soon as time reaches 120, status should have been toggled to IDLE.
	 * (3) After some arbitrary time (until 320), the status should remain IDLE, and no watermarks should have been emitted.
	 * (4) Emit a record at 330. Status should become ACTIVE. This should schedule a idleness detection to be fired at 430.
	 * (5) Emit another record at 350 (which is before the next check). This should make the idleness check pass.
	 * (6) Advance time to 430 and trigger idleness detection. The status should still be ACTIVE due to step (5).
	 *     This should schedule a idleness detection to be fired at 530.
	 * (7) Advance time to 460, in which a watermark emission task should be fired. Idleness detection
	 *     should have been "piggy-backed" in the task, allowing the status to be toggled to IDLE before the next
	 *     actual idle detection task at 530.
	 *
	 * Inline comments will refer to the corresponding tested steps in the scenario.
	 */
@Test
public void testAutomaticWatermarkContext() throws Exception {
    long watermarkInterval = 40;
    long idleTimeout = 100;
    long initialTime = 20;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(initialTime);
    MockStreamStatusMaintainer mockStreamStatusMaintainer = new MockStreamStatusMaintainer();
    final List<StreamElement> output = new ArrayList<>();
    final List<StreamElement> expectedOutput = new ArrayList<>();
    SourceFunction.SourceContext<String> context = StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime, processingTimeService, new Object(), mockStreamStatusMaintainer, new CollectorOutput<String>(output), watermarkInterval, idleTimeout);
    // -------------------------- begin test scenario --------------------------
    // corresponds to step (2) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + watermarkInterval);
    expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
    processingTimeService.setCurrentTime(initialTime + 2 * watermarkInterval);
    expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
    processingTimeService.setCurrentTime(initialTime + idleTimeout);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
    assertEquals(expectedOutput, output);
    // corresponds to step (3) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 4 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout);
    processingTimeService.setCurrentTime(initialTime + 6 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 7 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
    assertEquals(expectedOutput, output);
    // corresponds to step (4) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            context.collect("msg");
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
            assertEquals(expectedOutput, output);
            break;
        case COLLECT_WITH_TIMESTAMP:
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            // for emitWatermark, since the watermark will be blocked,
            // it should not make the status become active;
            // from here on, the status should remain idle for the emitWatermark variant test
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
            assertEquals(expectedOutput, output);
    }
    // corresponds to step (5) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 8 * watermarkInterval);
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + 3 * idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            context.collect("msg");
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
            assertEquals(expectedOutput, output);
            break;
        case COLLECT_WITH_TIMESTAMP:
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            expectedOutput.add(new StreamRecord<>("msg", processingTimeService.getCurrentProcessingTime()));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
            assertEquals(expectedOutput, output);
    }
    processingTimeService.setCurrentTime(initialTime + 9 * watermarkInterval);
    switch(testMethod) {
        case COLLECT:
        case COLLECT_WITH_TIMESTAMP:
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
            assertEquals(expectedOutput, output);
    }
    processingTimeService.setCurrentTime(initialTime + 10 * watermarkInterval);
    switch(testMethod) {
        case COLLECT:
        case COLLECT_WITH_TIMESTAMP:
            expectedOutput.add(new Watermark(processingTimeService.getCurrentProcessingTime() - (processingTimeService.getCurrentProcessingTime() % watermarkInterval)));
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
            assertEquals(expectedOutput, output);
    }
    // corresponds to step (6) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 4 * idleTimeout + idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
        case COLLECT_WITH_TIMESTAMP:
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
            assertEquals(expectedOutput, output);
            break;
        case EMIT_WATERMARK:
            assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
            assertEquals(expectedOutput, output);
    }
    // corresponds to step (7) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 11 * watermarkInterval);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
    assertEquals(expectedOutput, output);
}

Example 43 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class StreamSourceContextIdleDetectionTests method testManualWatermarkContext.

/**
	 * Test scenario (idleTimeout = 100):
	 * (1) Start from 0 as initial time.
	 * (2) As soon as time reaches 100, status should have been toggled to IDLE.
	 * (3) After some arbitrary time (until 300), the status should remain IDLE.
	 * (4) Emit a record at 310. Status should become ACTIVE. This should fire a idleness detection at 410.
	 * (5) Emit another record at 320 (which is before the next check). This should make the idleness check pass.
	 * (6) Advance time to 410 and trigger idleness detection.
	 *     The status should still be ACTIVE due to step (5). Another idleness detection should be fired at 510.
	 * (7) Advance time to 510 and trigger idleness detection. Since no records were collected in-between the two
	 *     idleness detections, status should have been toggle back to IDLE.
	 *
	 * Inline comments will refer to the corresponding tested steps in the scenario.
	 */
@Test
public void testManualWatermarkContext() throws Exception {
    long idleTimeout = 100;
    long initialTime = 0;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(initialTime);
    final List<StreamElement> output = new ArrayList<>();
    MockStreamStatusMaintainer mockStreamStatusMaintainer = new MockStreamStatusMaintainer();
    SourceFunction.SourceContext<String> context = StreamSourceContexts.getSourceContext(TimeCharacteristic.EventTime, processingTimeService, new Object(), mockStreamStatusMaintainer, new CollectorOutput<String>(output), 0, idleTimeout);
    // -------------------------- begin test scenario --------------------------
    // corresponds to step (2) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + idleTimeout);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
    // corresponds to step (3) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout);
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
    // corresponds to step (4) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            context.collect("msg");
            break;
        case COLLECT_WITH_TIMESTAMP:
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            break;
        case EMIT_WATERMARK:
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            break;
    }
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
    // corresponds to step (5) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + 2 * idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            context.collect("msg");
            break;
        case COLLECT_WITH_TIMESTAMP:
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            break;
        case EMIT_WATERMARK:
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            break;
    }
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
    // corresponds to step (6) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 4 * idleTimeout + idleTimeout / 10);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
    // corresponds to step (7) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 5 * idleTimeout + idleTimeout / 10);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
}

Example 44 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class CoStreamMapTest method testCoMap.

@Test
public void testCoMap() throws Exception {
    CoStreamMap<Double, Integer, String> operator = new CoStreamMap<Double, Integer, String>(new MyCoMap());
    TwoInputStreamOperatorTestHarness<Double, Integer, String> testHarness = new TwoInputStreamOperatorTestHarness<Double, Integer, String>(operator);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    testHarness.open();
    testHarness.processElement1(new StreamRecord<Double>(1.1d, initialTime + 1));
    testHarness.processElement1(new StreamRecord<Double>(1.2d, initialTime + 2));
    testHarness.processElement1(new StreamRecord<Double>(1.3d, initialTime + 3));
    testHarness.processWatermark1(new Watermark(initialTime + 3));
    testHarness.processElement1(new StreamRecord<Double>(1.4d, initialTime + 4));
    testHarness.processElement1(new StreamRecord<Double>(1.5d, initialTime + 5));
    testHarness.processElement2(new StreamRecord<Integer>(1, initialTime + 1));
    testHarness.processElement2(new StreamRecord<Integer>(2, initialTime + 2));
    testHarness.processWatermark2(new Watermark(initialTime + 2));
    testHarness.processElement2(new StreamRecord<Integer>(3, initialTime + 3));
    testHarness.processElement2(new StreamRecord<Integer>(4, initialTime + 4));
    testHarness.processElement2(new StreamRecord<Integer>(5, initialTime + 5));
    expectedOutput.add(new StreamRecord<String>("1.1", initialTime + 1));
    expectedOutput.add(new StreamRecord<String>("1.2", initialTime + 2));
    expectedOutput.add(new StreamRecord<String>("1.3", initialTime + 3));
    expectedOutput.add(new StreamRecord<String>("1.4", initialTime + 4));
    expectedOutput.add(new StreamRecord<String>("1.5", initialTime + 5));
    expectedOutput.add(new StreamRecord<String>("1", initialTime + 1));
    expectedOutput.add(new StreamRecord<String>("2", initialTime + 2));
    expectedOutput.add(new Watermark(initialTime + 2));
    expectedOutput.add(new StreamRecord<String>("3", initialTime + 3));
    expectedOutput.add(new StreamRecord<String>("4", initialTime + 4));
    expectedOutput.add(new StreamRecord<String>("5", initialTime + 5));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 45 with Watermark

use of org.apache.flink.streaming.api.watermark.Watermark in project flink by apache.

the class KeyedCoProcessOperatorTest method testEventTimeTimerWithState.

/**
	 * Verifies that we don't have leakage between different keys.
	 */
@Test
public void testEventTimeTimerWithState() throws Exception {
    KeyedCoProcessOperator<String, Integer, String, String> operator = new KeyedCoProcessOperator<>(new EventTimeTriggeringStatefulProcessFunction());
    TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>(operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setup();
    testHarness.open();
    testHarness.processWatermark1(new Watermark(1));
    testHarness.processWatermark2(new Watermark(1));
    // should set timer for 6
    testHarness.processElement1(new StreamRecord<>(17, 0L));
    testHarness.processWatermark1(new Watermark(2));
    testHarness.processWatermark2(new Watermark(2));
    // should set timer for 7
    testHarness.processElement2(new StreamRecord<>("42", 1L));
    testHarness.processWatermark1(new Watermark(6));
    testHarness.processWatermark2(new Watermark(6));
    testHarness.processWatermark1(new Watermark(7));
    testHarness.processWatermark2(new Watermark(7));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(1L));
    expectedOutput.add(new StreamRecord<>("INPUT1:17", 0L));
    expectedOutput.add(new Watermark(2L));
    expectedOutput.add(new StreamRecord<>("INPUT2:42", 1L));
    expectedOutput.add(new StreamRecord<>("STATE:17", 6L));
    expectedOutput.add(new Watermark(6L));
    expectedOutput.add(new StreamRecord<>("STATE:42", 7L));
    expectedOutput.add(new Watermark(7L));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.close();
}