Examples with SourceEvent org.apache.flink.api.connector.source.SourceEvent used on opensource projects

Example 1 with SourceEvent

use of org.apache.flink.api.connector.source.SourceEvent in project flink by apache.

the class SourceCoordinator method handleEventFromOperator.

@Override
public void handleEventFromOperator(int subtask, OperatorEvent event) {
    runInEventLoop(() -> {
        if (event instanceof RequestSplitEvent) {
            LOG.info("Source {} received split request from parallel task {}", operatorName, subtask);
            enumerator.handleSplitRequest(subtask, ((RequestSplitEvent) event).hostName());
        } else if (event instanceof SourceEventWrapper) {
            final SourceEvent sourceEvent = ((SourceEventWrapper) event).getSourceEvent();
            LOG.debug("Source {} received custom event from parallel task {}: {}", operatorName, subtask, sourceEvent);
            enumerator.handleSourceEvent(subtask, sourceEvent);
        } else if (event instanceof ReaderRegistrationEvent) {
            final ReaderRegistrationEvent registrationEvent = (ReaderRegistrationEvent) event;
            LOG.info("Source {} registering reader for parallel task {} @ {}", operatorName, subtask, registrationEvent.location());
            handleReaderRegistrationEvent(registrationEvent);
        } else if (event instanceof ReportedWatermarkEvent) {
            handleReportedWatermark(subtask, new Watermark(((ReportedWatermarkEvent) event).getWatermark()));
        } else {
            throw new FlinkException("Unrecognized Operator Event: " + event);
        }
    }, "handling operator event %s from subtask %d", event, subtask);
}

Example 2 with SourceEvent

use of org.apache.flink.api.connector.source.SourceEvent in project flink by apache.

the class SourceCoordinatorTest method testHandleSourceEvent.

@Test
public void testHandleSourceEvent() throws Exception {
    sourceReady();
    SourceEvent sourceEvent = new SourceEvent() {
    };
    sourceCoordinator.handleEventFromOperator(0, new SourceEventWrapper(sourceEvent));
    waitForCoordinatorToProcessActions();
    assertEquals(1, getEnumerator().getHandledSourceEvent().size());
    assertEquals(sourceEvent, getEnumerator().getHandledSourceEvent().get(0));
}

Example 3 with SourceEvent

use of org.apache.flink.api.connector.source.SourceEvent in project flink by apache.

the class SourceCoordinatorTest method testErrorThrownFromSplitEnumerator.

@Test
public void testErrorThrownFromSplitEnumerator() throws Exception {
    final Error error = new Error("Test Error");
    try (final MockSplitEnumeratorContext<MockSourceSplit> enumeratorContext = new MockSplitEnumeratorContext<>(1);
        final SplitEnumerator<MockSourceSplit, Set<MockSourceSplit>> splitEnumerator = new MockSplitEnumerator(1, enumeratorContext) {

            @Override
            public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) {
                throw error;
            }
        };
        final SourceCoordinator<?, ?> coordinator = new SourceCoordinator<>(OPERATOR_NAME, new EnumeratorCreatingSource<>(() -> splitEnumerator), context, new CoordinatorStoreImpl(), WatermarkAlignmentParams.WATERMARK_ALIGNMENT_DISABLED)) {
        coordinator.start();
        coordinator.handleEventFromOperator(1, new SourceEventWrapper(new SourceEvent() {
        }));
        waitUtil(() -> operatorCoordinatorContext.isJobFailed(), Duration.ofSeconds(10), "The job should have failed due to the artificial exception.");
        assertEquals(error, operatorCoordinatorContext.getJobFailureReason());
    }
}

Example 4 with SourceEvent

use of org.apache.flink.api.connector.source.SourceEvent in project flink by apache.

the class SourceCoordinatorTest method testBlockOnClose.

@Test
public void testBlockOnClose() throws Exception {
    // It is possible that the split enumerator submits some heavy-duty work to the
    // coordinator executor which blocks the coordinator closure.
    final CountDownLatch latch = new CountDownLatch(1);
    try (final MockSplitEnumeratorContext<MockSourceSplit> enumeratorContext = new MockSplitEnumeratorContext<>(1);
        final MockSplitEnumerator splitEnumerator = new MockSplitEnumerator(1, enumeratorContext) {

            @Override
            public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) {
                context.callAsync(() -> 1L, (ignored, t) -> {
                    latch.countDown();
                    // Submit a callable that will never return.
                    try {
                        Thread.sleep(Long.MAX_VALUE);
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    }
                });
            }
        };
        final SourceCoordinator<?, ?> coordinator = new SourceCoordinator<>(OPERATOR_NAME, new EnumeratorCreatingSource<>(() -> splitEnumerator), context, new CoordinatorStoreImpl())) {
        coordinator.start();
        coordinator.handleEventFromOperator(1, new SourceEventWrapper(new SourceEvent() {
        }));
        // Wait until the coordinator executor blocks.
        latch.await();
        CompletableFuture<?> future = ComponentClosingUtils.closeAsyncWithTimeout("testBlockOnClose", (ThrowingRunnable<Exception>) coordinator::close, Duration.ofMillis(1));
        future.exceptionally(e -> {
            assertTrue(e instanceof TimeoutException);
            return null;
        }).get();
        waitUtil(splitEnumerator::closed, Duration.ofSeconds(5), "Split enumerator was not closed in 5 seconds.");
    }
}

Example 5 with SourceEvent

use of org.apache.flink.api.connector.source.SourceEvent in project flink by apache.

the class SourceOperator method initReader.

/**
 * Initializes the reader. The code from this method should ideally happen in the constructor or
 * in the operator factory even. It has to happen here at a slightly later stage, because of the
 * lazy metric initialization.
 *
 * <p>Calling this method explicitly is an optional way to have the reader initialization a bit
 * earlier than in open(), as needed by the {@link
 * org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask}
 *
 * <p>This code should move to the constructor once the metric groups are available at task
 * setup time.
 */
public void initReader() throws Exception {
    if (sourceReader != null) {
        return;
    }
    final int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
    final SourceReaderContext context = new SourceReaderContext() {

        @Override
        public SourceReaderMetricGroup metricGroup() {
            return sourceMetricGroup;
        }

        @Override
        public Configuration getConfiguration() {
            return configuration;
        }

        @Override
        public String getLocalHostName() {
            return localHostname;
        }

        @Override
        public int getIndexOfSubtask() {
            return subtaskIndex;
        }

        @Override
        public void sendSplitRequest() {
            operatorEventGateway.sendEventToCoordinator(new RequestSplitEvent(getLocalHostName()));
        }

        @Override
        public void sendSourceEventToCoordinator(SourceEvent event) {
            operatorEventGateway.sendEventToCoordinator(new SourceEventWrapper(event));
        }

        @Override
        public UserCodeClassLoader getUserCodeClassLoader() {
            return new UserCodeClassLoader() {

                @Override
                public ClassLoader asClassLoader() {
                    return getRuntimeContext().getUserCodeClassLoader();
                }

                @Override
                public void registerReleaseHookIfAbsent(String releaseHookName, Runnable releaseHook) {
                    getRuntimeContext().registerUserCodeClassLoaderReleaseHookIfAbsent(releaseHookName, releaseHook);
                }
            };
        }
    };
    sourceReader = readerFactory.apply(context);
}