Examples with WatermarkAlignmentEvent org.apache.flink.runtime.source.event.WatermarkAlignmentEvent used on opensource projects

Example 1 with WatermarkAlignmentEvent

use of org.apache.flink.runtime.source.event.WatermarkAlignmentEvent in project flink by apache.

the class SourceCoordinatorAlignmentTest method assertLatestWatermarkAlignmentEvent.

private void assertLatestWatermarkAlignmentEvent(int subtask, long expectedWatermark) {
    List<OperatorEvent> events = receivingTasks.getSentEventsForSubtask(subtask);
    assertFalse(events.isEmpty());
    assertEquals(new WatermarkAlignmentEvent(expectedWatermark), events.get(events.size() - 1));
}

Example 2 with WatermarkAlignmentEvent

use of org.apache.flink.runtime.source.event.WatermarkAlignmentEvent in project flink by apache.

the class SourceCoordinator method announceCombinedWatermark.

@VisibleForTesting
void announceCombinedWatermark() {
    checkState(watermarkAlignmentParams != WatermarkAlignmentParams.WATERMARK_ALIGNMENT_DISABLED);
    Watermark globalCombinedWatermark = coordinatorStore.apply(watermarkAlignmentParams.getWatermarkGroup(), (value) -> {
        WatermarkAggregator aggregator = (WatermarkAggregator) value;
        return new Watermark(aggregator.getAggregatedWatermark().getTimestamp());
    });
    long maxAllowedWatermark = globalCombinedWatermark.getTimestamp() + watermarkAlignmentParams.getMaxAllowedWatermarkDrift();
    Set<Integer> subTaskIds = combinedWatermark.keySet();
    LOG.info("Distributing maxAllowedWatermark={} to subTaskIds={}", maxAllowedWatermark, subTaskIds);
    for (Integer subtaskId : subTaskIds) {
        context.sendEventToSourceOperator(subtaskId, new WatermarkAlignmentEvent(maxAllowedWatermark));
    }
}

Example 3 with WatermarkAlignmentEvent

use of org.apache.flink.runtime.source.event.WatermarkAlignmentEvent in project flink by apache.

the class SourceOperatorAlignmentTest method testWatermarkAlignment.

@Test
public void testWatermarkAlignment() throws Exception {
    operator.initializeState(context.createStateContext());
    operator.open();
    MockSourceSplit newSplit = new MockSourceSplit(2);
    int record1 = 1000;
    int record2 = 2000;
    int record3 = 3000;
    newSplit.addRecord(record1);
    newSplit.addRecord(record2);
    newSplit.addRecord(record3);
    operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
    CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
    List<Integer> expectedOutput = new ArrayList<>();
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
    expectedOutput.add(record1);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record1);
    assertOutput(actualOutput, expectedOutput);
    assertTrue(operator.isAvailable());
    operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 - 1));
    assertFalse(operator.isAvailable());
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
    assertLatestReportedWatermarkEvent(record1);
    assertOutput(actualOutput, expectedOutput);
    assertFalse(operator.isAvailable());
    operator.handleOperatorEvent(new WatermarkAlignmentEvent(record1 + 1));
    assertTrue(operator.isAvailable());
    operator.emitNext(actualOutput);
    // Try to poll a record second time. Technically speaking previous emitNext call could have
    // already switch the operator status to unavailable, but that's an implementation detail.
    // However, this second call can not emit anything and should after that second call
    // operator must be unavailable.
    assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
    expectedOutput.add(record2);
    context.getTimeService().advance(1);
    assertLatestReportedWatermarkEvent(record2);
    assertOutput(actualOutput, expectedOutput);
    assertFalse(operator.isAvailable());
}