Examples with Watermark org.apache.flink.api.common.eventtime.Watermark used on opensource projects

Example 1 with Watermark

use of org.apache.flink.api.common.eventtime.Watermark in project flink by apache.

the class SourceCoordinator method handleEventFromOperator.

@Override
public void handleEventFromOperator(int subtask, OperatorEvent event) {
    runInEventLoop(() -> {
        if (event instanceof RequestSplitEvent) {
            LOG.info("Source {} received split request from parallel task {}", operatorName, subtask);
            enumerator.handleSplitRequest(subtask, ((RequestSplitEvent) event).hostName());
        } else if (event instanceof SourceEventWrapper) {
            final SourceEvent sourceEvent = ((SourceEventWrapper) event).getSourceEvent();
            LOG.debug("Source {} received custom event from parallel task {}: {}", operatorName, subtask, sourceEvent);
            enumerator.handleSourceEvent(subtask, sourceEvent);
        } else if (event instanceof ReaderRegistrationEvent) {
            final ReaderRegistrationEvent registrationEvent = (ReaderRegistrationEvent) event;
            LOG.info("Source {} registering reader for parallel task {} @ {}", operatorName, subtask, registrationEvent.location());
            handleReaderRegistrationEvent(registrationEvent);
        } else if (event instanceof ReportedWatermarkEvent) {
            handleReportedWatermark(subtask, new Watermark(((ReportedWatermarkEvent) event).getWatermark()));
        } else {
            throw new FlinkException("Unrecognized Operator Event: " + event);
        }
    }, "handling operator event %s from subtask %d", event, subtask);
}

Example 2 with Watermark

use of org.apache.flink.api.common.eventtime.Watermark in project flink by apache.

the class SourceCoordinator method announceCombinedWatermark.

@VisibleForTesting
void announceCombinedWatermark() {
    checkState(watermarkAlignmentParams != WatermarkAlignmentParams.WATERMARK_ALIGNMENT_DISABLED);
    Watermark globalCombinedWatermark = coordinatorStore.apply(watermarkAlignmentParams.getWatermarkGroup(), (value) -> {
        WatermarkAggregator aggregator = (WatermarkAggregator) value;
        return new Watermark(aggregator.getAggregatedWatermark().getTimestamp());
    });
    long maxAllowedWatermark = globalCombinedWatermark.getTimestamp() + watermarkAlignmentParams.getMaxAllowedWatermarkDrift();
    Set<Integer> subTaskIds = combinedWatermark.keySet();
    LOG.info("Distributing maxAllowedWatermark={} to subTaskIds={}", maxAllowedWatermark, subTaskIds);
    for (Integer subtaskId : subTaskIds) {
        context.sendEventToSourceOperator(subtaskId, new WatermarkAlignmentEvent(maxAllowedWatermark));
    }
}

Example 3 with Watermark

use of org.apache.flink.api.common.eventtime.Watermark in project flink by apache.

the class SourceOperatorAlignmentTest method testWatermarkAlignmentWithIdleness.

@Test
public void testWatermarkAlignmentWithIdleness() throws Exception {
    // we use a separate context, because we need to enable idleness
    try (SourceOperatorTestContext context = new SourceOperatorTestContext(true, WatermarkStrategy.forGenerator(ctx -> new PunctuatedGenerator(PunctuatedGenerator.GenerationMode.ODD)).withWatermarkAlignment("group1", Duration.ofMillis(100), Duration.ofMillis(1)).withTimestampAssigner((r, t) -> r))) {
        final SourceOperator<Integer, MockSourceSplit> operator = context.getOperator();
        operator.initializeState(context.createStateContext());
        operator.open();
        MockSourceSplit newSplit = new MockSourceSplit(2);
        int record1 = 1;
        newSplit.addRecord(record1);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        CollectingDataOutput<Integer> actualOutput = new CollectingDataOutput<>();
        List<Integer> expectedOutput = new ArrayList<>();
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record1);
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
        // source becomes idle, it should report Long.MAX_VALUE as the watermark
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.NOTHING_AVAILABLE));
        context.getTimeService().advance(1);
        assertLatestReportedWatermarkEvent(context, Long.MAX_VALUE);
        // it is easier to create a new split than add records the old one. The old one is
        // serialized, when sending the AddSplitEvent, so it is not as easy as
        // newSplit.addRecord
        newSplit = new MockSourceSplit(3);
        // even timestamp -> no watermarks
        int record2 = 2;
        newSplit.addRecord(record2);
        operator.handleOperatorEvent(new AddSplitEvent<>(Collections.singletonList(newSplit), new MockSourceSplitSerializer()));
        assertThat(operator.emitNext(actualOutput), is(DataInputStatus.MORE_AVAILABLE));
        expectedOutput.add(record2);
        context.getTimeService().advance(1);
        // becomes active again, should go back to the previously emitted
        // watermark, as the record2 does not emit watermarks
        assertLatestReportedWatermarkEvent(context, record1);
        assertOutput(actualOutput, expectedOutput);
        assertTrue(operator.isAvailable());
    }
}