use of org.apache.flink.runtime.source.event.SourceEventWrapper in project flink by apache.
the class SourceCoordinator method handleEventFromOperator.
@Override
public void handleEventFromOperator(int subtask, OperatorEvent event) {
runInEventLoop(() -> {
if (event instanceof RequestSplitEvent) {
LOG.info("Source {} received split request from parallel task {}", operatorName, subtask);
enumerator.handleSplitRequest(subtask, ((RequestSplitEvent) event).hostName());
} else if (event instanceof SourceEventWrapper) {
final SourceEvent sourceEvent = ((SourceEventWrapper) event).getSourceEvent();
LOG.debug("Source {} received custom event from parallel task {}: {}", operatorName, subtask, sourceEvent);
enumerator.handleSourceEvent(subtask, sourceEvent);
} else if (event instanceof ReaderRegistrationEvent) {
final ReaderRegistrationEvent registrationEvent = (ReaderRegistrationEvent) event;
LOG.info("Source {} registering reader for parallel task {} @ {}", operatorName, subtask, registrationEvent.location());
handleReaderRegistrationEvent(registrationEvent);
} else if (event instanceof ReportedWatermarkEvent) {
handleReportedWatermark(subtask, new Watermark(((ReportedWatermarkEvent) event).getWatermark()));
} else {
throw new FlinkException("Unrecognized Operator Event: " + event);
}
}, "handling operator event %s from subtask %d", event, subtask);
}
use of org.apache.flink.runtime.source.event.SourceEventWrapper in project flink by apache.
the class SourceCoordinatorTest method testHandleSourceEvent.
@Test
public void testHandleSourceEvent() throws Exception {
sourceReady();
SourceEvent sourceEvent = new SourceEvent() {
};
sourceCoordinator.handleEventFromOperator(0, new SourceEventWrapper(sourceEvent));
waitForCoordinatorToProcessActions();
assertEquals(1, getEnumerator().getHandledSourceEvent().size());
assertEquals(sourceEvent, getEnumerator().getHandledSourceEvent().get(0));
}
use of org.apache.flink.runtime.source.event.SourceEventWrapper in project flink by apache.
the class SourceCoordinatorTest method testErrorThrownFromSplitEnumerator.
@Test
public void testErrorThrownFromSplitEnumerator() throws Exception {
final Error error = new Error("Test Error");
try (final MockSplitEnumeratorContext<MockSourceSplit> enumeratorContext = new MockSplitEnumeratorContext<>(1);
final SplitEnumerator<MockSourceSplit, Set<MockSourceSplit>> splitEnumerator = new MockSplitEnumerator(1, enumeratorContext) {
@Override
public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) {
throw error;
}
};
final SourceCoordinator<?, ?> coordinator = new SourceCoordinator<>(OPERATOR_NAME, new EnumeratorCreatingSource<>(() -> splitEnumerator), context, new CoordinatorStoreImpl(), WatermarkAlignmentParams.WATERMARK_ALIGNMENT_DISABLED)) {
coordinator.start();
coordinator.handleEventFromOperator(1, new SourceEventWrapper(new SourceEvent() {
}));
waitUtil(() -> operatorCoordinatorContext.isJobFailed(), Duration.ofSeconds(10), "The job should have failed due to the artificial exception.");
assertEquals(error, operatorCoordinatorContext.getJobFailureReason());
}
}
use of org.apache.flink.runtime.source.event.SourceEventWrapper in project flink by apache.
the class SourceCoordinatorTest method testBlockOnClose.
@Test
public void testBlockOnClose() throws Exception {
// It is possible that the split enumerator submits some heavy-duty work to the
// coordinator executor which blocks the coordinator closure.
final CountDownLatch latch = new CountDownLatch(1);
try (final MockSplitEnumeratorContext<MockSourceSplit> enumeratorContext = new MockSplitEnumeratorContext<>(1);
final MockSplitEnumerator splitEnumerator = new MockSplitEnumerator(1, enumeratorContext) {
@Override
public void handleSourceEvent(int subtaskId, SourceEvent sourceEvent) {
context.callAsync(() -> 1L, (ignored, t) -> {
latch.countDown();
// Submit a callable that will never return.
try {
Thread.sleep(Long.MAX_VALUE);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
});
}
};
final SourceCoordinator<?, ?> coordinator = new SourceCoordinator<>(OPERATOR_NAME, new EnumeratorCreatingSource<>(() -> splitEnumerator), context, new CoordinatorStoreImpl())) {
coordinator.start();
coordinator.handleEventFromOperator(1, new SourceEventWrapper(new SourceEvent() {
}));
// Wait until the coordinator executor blocks.
latch.await();
CompletableFuture<?> future = ComponentClosingUtils.closeAsyncWithTimeout("testBlockOnClose", (ThrowingRunnable<Exception>) coordinator::close, Duration.ofMillis(1));
future.exceptionally(e -> {
assertTrue(e instanceof TimeoutException);
return null;
}).get();
waitUtil(splitEnumerator::closed, Duration.ofSeconds(5), "Split enumerator was not closed in 5 seconds.");
}
}
use of org.apache.flink.runtime.source.event.SourceEventWrapper in project flink by apache.
the class SourceOperator method initReader.
/**
* Initializes the reader. The code from this method should ideally happen in the constructor or
* in the operator factory even. It has to happen here at a slightly later stage, because of the
* lazy metric initialization.
*
* <p>Calling this method explicitly is an optional way to have the reader initialization a bit
* earlier than in open(), as needed by the {@link
* org.apache.flink.streaming.runtime.tasks.SourceOperatorStreamTask}
*
* <p>This code should move to the constructor once the metric groups are available at task
* setup time.
*/
public void initReader() throws Exception {
if (sourceReader != null) {
return;
}
final int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
final SourceReaderContext context = new SourceReaderContext() {
@Override
public SourceReaderMetricGroup metricGroup() {
return sourceMetricGroup;
}
@Override
public Configuration getConfiguration() {
return configuration;
}
@Override
public String getLocalHostName() {
return localHostname;
}
@Override
public int getIndexOfSubtask() {
return subtaskIndex;
}
@Override
public void sendSplitRequest() {
operatorEventGateway.sendEventToCoordinator(new RequestSplitEvent(getLocalHostName()));
}
@Override
public void sendSourceEventToCoordinator(SourceEvent event) {
operatorEventGateway.sendEventToCoordinator(new SourceEventWrapper(event));
}
@Override
public UserCodeClassLoader getUserCodeClassLoader() {
return new UserCodeClassLoader() {
@Override
public ClassLoader asClassLoader() {
return getRuntimeContext().getUserCodeClassLoader();
}
@Override
public void registerReleaseHookIfAbsent(String releaseHookName, Runnable releaseHook) {
getRuntimeContext().registerUserCodeClassLoaderReleaseHookIfAbsent(releaseHookName, releaseHook);
}
};
}
};
sourceReader = readerFactory.apply(context);
}
Aggregations