Search in sources :

Example 1 with CollectingDataOutput

use of org.apache.flink.streaming.api.operators.source.CollectingDataOutput in project flink by apache.

the class SourceOperatorAlignmentTest method testReportedWatermarkDoNotDecrease.

@Test
public void testReportedWatermarkDoNotDecrease() throws Exception {
    operator.initializeState(context.createStateContext());
    operator.open();
    MockSourceSplit split1 = new MockSourceSplit(2);
    MockSourceSplit split2 = new MockSourceSplit(3);
    int record1 = 2000;
    int record2 = 1000;
    split1.addRecord(record1);
    split2.addRecord(record2);
    operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(split1), new MockSourceSplitSerializer()));
    CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
    operator.emitNext(actualOutput);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record1);
    operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(split2), new MockSourceSplitSerializer()));
    operator.emitNext(actualOutput);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record1);
}
Also used : CollectingDataOutput(org.apache.flink.streaming.api.operators.source.CollectingDataOutput) MockSourceSplitSerializer(org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer) MockSourceSplit(org.apache.flink.api.connector.source.mocks.MockSourceSplit) Test(org.junit.jupiter.api.Test)

Example 2 with CollectingDataOutput

use of org.apache.flink.streaming.api.operators.source.CollectingDataOutput in project flink by apache.

the class SourceOperatorAlignmentTest method testWatermarkAlignment.

@Test
public void testWatermarkAlignment() throws Exception {
    operator.initializeState(context.createStateContext());
    operator.open();
    MockSourceSplit newSplit = new MockSourceSplit(2);
    int record1 = 1000;
    int record2 = 2000;
    int record3 = 3000;
    newSplit.addRecord(record1);
    newSplit.addRecord(record2);
    newSplit.addRecord(record3);
    operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
    CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
    List<Integer> expectedOutput = new ArrayList<>();
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
    expectedOutput.add(record1);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record1);
    assertOutput(actualOutput, expectedOutput);
    assertTrue(operator.isAvailable());
    operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 - 1));
    assertFalse(operator.isAvailable());
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
    assertLatestReportedWatermarkEvent(record1);
    assertOutput(actualOutput, expectedOutput);
    assertFalse(operator.isAvailable());
    operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 + 1));
    assertTrue(operator.isAvailable());
    operator.emitNext(actualOutput);
    // Try to poll a record second time. Technically speaking previous emitNext call could have
    // already switch the operator status to unavailable, but that's an implementation detail.
    // However, this second call can not emit anything and should after that second call
    // operator must be unavailable.
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
    expectedOutput.add(record2);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record2);
    assertOutput(actualOutput, expectedOutput);
    assertFalse(operator.isAvailable());
}
Also used : ArrayList(java.util.ArrayList) CollectingDataOutput(org.apache.flink.streaming.api.operators.source.CollectingDataOutput) MockSourceSplitSerializer(org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer) WatermarkAlignmentEvent(org.apache.flink.runtime.source.event.WatermarkAlignmentEvent) MockSourceSplit(org.apache.flink.api.connector.source.mocks.MockSourceSplit) Test(org.junit.jupiter.api.Test)

Example 3 with CollectingDataOutput

use of org.apache.flink.streaming.api.operators.source.CollectingDataOutput in project flink by apache.

the class SourceOperatorAlignmentTest method testWatermarkAlignmentWithIdleness.

@Test
public void testWatermarkAlignmentWithIdleness() throws Exception {
    // we use a separate context, because we need to enable idleness
    try (SourceOperatorTestContext context = new SourceOperatorTestContext(true, WatermarkStrategy.forGenerator(ctx -> new PunctuatedGenerator(PunctuatedGenerator.GenerationMode.ODD)).withWatermarkAlignment("group1", Duration.ofMillis(100), Duration.ofMillis(1)).withTimestampAssigner((r, t) -> r))) {
        final SourceOperator<Integer, MockSourceSplit> operator = context.getOperator();
        operator.initializeState(context.createStateContext());
        operator.open();
        MockSourceSplit newSplit = new MockSourceSplit(2);
        int record1 = 1;
        newSplit.addRecord(record1);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
        List<Integer> expectedOutput = new ArrayList<>();
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record1);
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
        // source becomes idle, it should report Long.MAX_VALUE as the watermark
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, Long.MAX_VALUE);
        // it is easier to create a new split than add records the old one. The old one is
        // serialized, when sending the AddSplitEvent, so it is not as easy as
        // newSplit.addRecord
        newSplit = new MockSourceSplit(3);
        // even timestamp -> no watermarks
        int record2 = 2;
        newSplit.addRecord(record2);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record2);
        context.getTimeService().advance(1);
        // becomes active again, should go back to the previously emitted
        // watermark, as the record2 does not emit watermarks
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
    }
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) BeforeEach(org.junit.jupiter.api.BeforeEach) ReportedWatermarkEvent(org.apache.flink.runtime.source.event.ReportedWatermarkEvent) WatermarkGenerator(org.apache.flink.api.common.eventtime.WatermarkGenerator) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) MockSourceSplit(org.apache.flink.api.connector.source.mocks.MockSourceSplit) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) DataInputStatus(org.apache.flink.streaming.runtime.io.DataInputStatus) Duration(java.time.Duration) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) WatermarkAlignmentEvent(org.apache.flink.runtime.source.event.WatermarkAlignmentEvent) Nullable(javax.annotation.Nullable) WatermarkStrategy(org.apache.flink.api.common.eventtime.WatermarkStrategy) MockSourceSplitSerializer(org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer) AddSplitEvent(org.apache.flink.runtime.source.event.AddSplitEvent) WatermarkOutput(org.apache.flink.api.common.eventtime.WatermarkOutput) Collectors(java.util.stream.Collectors) StopMode(org.apache.flink.runtime.io.network.api.StopMode) Test(org.junit.jupiter.api.Test) AfterEach(org.junit.jupiter.api.AfterEach) List(java.util.List) Matchers.contains(org.hamcrest.Matchers.contains) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) Watermark(org.apache.flink.api.common.eventtime.Watermark) CollectingDataOutput(org.apache.flink.streaming.api.operators.source.CollectingDataOutput) Collections(java.util.Collections) OperatorEvent(org.apache.flink.runtime.operators.coordination.OperatorEvent) ArrayList(java.util.ArrayList) CollectingDataOutput(org.apache.flink.streaming.api.operators.source.CollectingDataOutput) MockSourceSplitSerializer(org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer) MockSourceSplit(org.apache.flink.api.connector.source.mocks.MockSourceSplit) Test(org.junit.jupiter.api.Test)

Aggregations

MockSourceSplit (org.apache.flink.api.connector.source.mocks.MockSourceSplit)3 MockSourceSplitSerializer (org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer)3 CollectingDataOutput (org.apache.flink.streaming.api.operators.source.CollectingDataOutput)3 Test (org.junit.jupiter.api.Test)3 ArrayList (java.util.ArrayList)2 WatermarkAlignmentEvent (org.apache.flink.runtime.source.event.WatermarkAlignmentEvent)2 Duration (java.time.Duration)1 Collections (java.util.Collections)1 List (java.util.List)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 Collectors (java.util.stream.Collectors)1 Nullable (javax.annotation.Nullable)1 Watermark (org.apache.flink.api.common.eventtime.Watermark)1 WatermarkGenerator (org.apache.flink.api.common.eventtime.WatermarkGenerator)1 WatermarkOutput (org.apache.flink.api.common.eventtime.WatermarkOutput)1 WatermarkStrategy (org.apache.flink.api.common.eventtime.WatermarkStrategy)1 StopMode (org.apache.flink.runtime.io.network.api.StopMode)1 OperatorEvent (org.apache.flink.runtime.operators.coordination.OperatorEvent)1 AddSplitEvent (org.apache.flink.runtime.source.event.AddSplitEvent)1 ReportedWatermarkEvent (org.apache.flink.runtime.source.event.ReportedWatermarkEvent)1