Examples with CollectingDataOutput org.apache.flink.streaming.api.operators.source.CollectingDataOutput used on opensource projects

Example 1 with CollectingDataOutput

use of org.apache.flink.streaming.api.operators.source.CollectingDataOutput in project flink by apache.

the class SourceOperatorAlignmentTest method testReportedWatermarkDoNotDecrease.

@Test
public void testReportedWatermarkDoNotDecrease() throws Exception {
    operator.initializeState(context.createStateContext());
    operator.open();
    MockSourceSplit split1 = new MockSourceSplit(2);
    MockSourceSplit split2 = new MockSourceSplit(3);
    int record1 = 2000;
    int record2 = 1000;
    split1.addRecord(record1);
    split2.addRecord(record2);
    operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(split1), new MockSourceSplitSerializer()));
    CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
    operator.emitNext(actualOutput);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record1);
    operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(split2), new MockSourceSplitSerializer()));
    operator.emitNext(actualOutput);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record1);
}

Example 2 with CollectingDataOutput

use of org.apache.flink.streaming.api.operators.source.CollectingDataOutput in project flink by apache.

the class SourceOperatorAlignmentTest method testWatermarkAlignment.

@Test
public void testWatermarkAlignment() throws Exception {
    operator.initializeState(context.createStateContext());
    operator.open();
    MockSourceSplit newSplit = new MockSourceSplit(2);
    int record1 = 1000;
    int record2 = 2000;
    int record3 = 3000;
    newSplit.addRecord(record1);
    newSplit.addRecord(record2);
    newSplit.addRecord(record3);
    operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
    CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
    List<Integer> expectedOutput = new ArrayList<>();
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
    expectedOutput.add(record1);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record1);
    assertOutput(actualOutput, expectedOutput);
    assertTrue(operator.isAvailable());
    operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 - 1));
    assertFalse(operator.isAvailable());
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
    assertLatestReportedWatermarkEvent(record1);
    assertOutput(actualOutput, expectedOutput);
    assertFalse(operator.isAvailable());
    operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 + 1));
    assertTrue(operator.isAvailable());
    operator.emitNext(actualOutput);
    // Try to poll a record second time. Technically speaking previous emitNext call could have
    // already switch the operator status to unavailable, but that's an implementation detail.
    // However, this second call can not emit anything and should after that second call
    // operator must be unavailable.
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
    expectedOutput.add(record2);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record2);
    assertOutput(actualOutput, expectedOutput);
    assertFalse(operator.isAvailable());
}

Example 3 with CollectingDataOutput

use of org.apache.flink.streaming.api.operators.source.CollectingDataOutput in project flink by apache.

the class SourceOperatorAlignmentTest method testWatermarkAlignmentWithIdleness.

@Test
public void testWatermarkAlignmentWithIdleness() throws Exception {
    // we use a separate context, because we need to enable idleness
    try (SourceOperatorTestContext context = new SourceOperatorTestContext(true, WatermarkStrategy.forGenerator(ctx -> new PunctuatedGenerator(PunctuatedGenerator.GenerationMode.ODD)).withWatermarkAlignment("group1", Duration.ofMillis(100), Duration.ofMillis(1)).withTimestampAssigner((r, t) -> r))) {
        final SourceOperator<Integer, MockSourceSplit> operator = context.getOperator();
        operator.initializeState(context.createStateContext());
        operator.open();
        MockSourceSplit newSplit = new MockSourceSplit(2);
        int record1 = 1;
        newSplit.addRecord(record1);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
        List<Integer> expectedOutput = new ArrayList<>();
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record1);
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
        // source becomes idle, it should report Long.MAX_VALUE as the watermark
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, Long.MAX_VALUE);
        // it is easier to create a new split than add records the old one. The old one is
        // serialized, when sending the AddSplitEvent, so it is not as easy as
        // newSplit.addRecord
        newSplit = new MockSourceSplit(3);
        // even timestamp -> no watermarks
        int record2 = 2;
        newSplit.addRecord(record2);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record2);
        context.getTimeService().advance(1);
        // becomes active again, should go back to the previously emitted
        // watermark, as the record2 does not emit watermarks
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
    }
}