use of org.apache.kafka.connect.source.SourceTaskContext in project debezium by debezium.
the class EmbeddedEngine method run.
/**
* Run this embedded connector and deliver database changes to the registered {@link Consumer}. This method blocks until
* the connector is stopped.
* <p>
* First, the method checks to see if this instance is currently {@link #run() running}, and if so immediately returns.
* <p>
* If the configuration is valid, this method starts the connector and starts polling the connector for change events.
* All messages are delivered in batches to the {@link Consumer} registered with this embedded connector. The batch size,
* polling
* frequency, and other parameters are controlled via configuration settings. This continues until this connector is
* {@link #stop() stopped}.
* <p>
* Note that there are two ways to stop a connector running on a thread: calling {@link #stop()} from another thread, or
* interrupting the thread (e.g., via {@link ExecutorService#shutdownNow()}).
* <p>
* This method can be called repeatedly as needed.
*/
@Override
public void run() {
if (runningThread.compareAndSet(null, Thread.currentThread())) {
final String engineName = config.getString(ENGINE_NAME);
final String connectorClassName = config.getString(CONNECTOR_CLASS);
final Optional<ConnectorCallback> connectorCallback = Optional.ofNullable(this.connectorCallback);
// Only one thread can be in this part of the method at a time ...
latch.countUp();
try {
if (!config.validateAndRecord(CONNECTOR_FIELDS, logger::error)) {
fail("Failed to start connector with invalid configuration (see logs for actual errors)");
return;
}
// Instantiate the connector ...
SourceConnector connector = null;
try {
@SuppressWarnings("unchecked") Class<? extends SourceConnector> connectorClass = (Class<SourceConnector>) classLoader.loadClass(connectorClassName);
connector = connectorClass.newInstance();
} catch (Throwable t) {
fail("Unable to instantiate connector class '" + connectorClassName + "'", t);
return;
}
// Instantiate the offset store ...
final String offsetStoreClassName = config.getString(OFFSET_STORAGE);
OffsetBackingStore offsetStore = null;
try {
@SuppressWarnings("unchecked") Class<? extends OffsetBackingStore> offsetStoreClass = (Class<OffsetBackingStore>) classLoader.loadClass(offsetStoreClassName);
offsetStore = offsetStoreClass.newInstance();
} catch (Throwable t) {
fail("Unable to instantiate OffsetBackingStore class '" + offsetStoreClassName + "'", t);
return;
}
// Initialize the offset store ...
try {
offsetStore.configure(workerConfig);
offsetStore.start();
} catch (Throwable t) {
fail("Unable to configure and start the '" + offsetStoreClassName + "' offset backing store", t);
return;
}
// Set up the offset commit policy ...
if (offsetCommitPolicy == null) {
offsetCommitPolicy = config.getInstance(EmbeddedEngine.OFFSET_COMMIT_POLICY, OffsetCommitPolicy.class, config);
}
// Initialize the connector using a context that does NOT respond to requests to reconfigure tasks ...
ConnectorContext context = new ConnectorContext() {
@Override
public void requestTaskReconfiguration() {
// Do nothing ...
}
@Override
public void raiseError(Exception e) {
fail(e.getMessage(), e);
}
};
connector.initialize(context);
OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetStore, engineName, keyConverter, valueConverter);
OffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetStore, engineName, keyConverter, valueConverter);
long commitTimeoutMs = config.getLong(OFFSET_COMMIT_TIMEOUT_MS);
try {
// Start the connector with the given properties and get the task configurations ...
connector.start(config.asMap());
connectorCallback.ifPresent(ConnectorCallback::connectorStarted);
List<Map<String, String>> taskConfigs = connector.taskConfigs(1);
Class<? extends Task> taskClass = connector.taskClass();
SourceTask task = null;
try {
task = (SourceTask) taskClass.newInstance();
} catch (IllegalAccessException | InstantiationException t) {
fail("Unable to instantiate connector's task class '" + taskClass.getName() + "'", t);
return;
}
try {
SourceTaskContext taskContext = () -> offsetReader;
task.initialize(taskContext);
task.start(taskConfigs.get(0));
connectorCallback.ifPresent(ConnectorCallback::taskStarted);
} catch (Throwable t) {
// Mask the passwords ...
Configuration config = Configuration.from(taskConfigs.get(0)).withMaskedPasswords();
String msg = "Unable to initialize and start connector's task class '" + taskClass.getName() + "' with config: " + config;
fail(msg, t);
return;
}
recordsSinceLastCommit = 0;
Throwable handlerError = null;
try {
timeOfLastCommitMillis = clock.currentTimeInMillis();
boolean keepProcessing = true;
List<SourceRecord> changeRecords = null;
while (runningThread.get() != null && handlerError == null && keepProcessing) {
try {
try {
logger.debug("Embedded engine is polling task for records on thread " + runningThread.get());
// blocks until there are values ...
changeRecords = task.poll();
logger.debug("Embedded engine returned from polling task for records");
} catch (InterruptedException e) {
// Interrupted while polling ...
logger.debug("Embedded engine interrupted on thread " + runningThread.get() + " while polling the task for records");
Thread.interrupted();
break;
}
try {
if (changeRecords != null && !changeRecords.isEmpty()) {
logger.debug("Received {} records from the task", changeRecords.size());
// First forward the records to the connector's consumer ...
for (SourceRecord record : changeRecords) {
try {
consumer.accept(record);
task.commitRecord(record);
} catch (StopConnectorException e) {
keepProcessing = false;
// Stop processing any more but first record the offset for this record's
// partition
offsetWriter.offset(record.sourcePartition(), record.sourceOffset());
recordsSinceLastCommit += 1;
break;
} catch (Throwable t) {
handlerError = t;
break;
}
// Record the offset for this record's partition
offsetWriter.offset(record.sourcePartition(), record.sourceOffset());
recordsSinceLastCommit += 1;
}
// Flush the offsets to storage if necessary ...
maybeFlush(offsetWriter, offsetCommitPolicy, commitTimeoutMs, task);
} else {
logger.debug("Received no records from the task");
}
} catch (Throwable t) {
// There was some sort of unexpected exception, so we should stop work
if (handlerError == null) {
// make sure we capture the error first so that we can report it later
handlerError = t;
}
break;
}
} finally {
// then try to commit the offsets, since we record them only after the records were handled
// by the consumer ...
maybeFlush(offsetWriter, offsetCommitPolicy, commitTimeoutMs, task);
}
}
} finally {
if (handlerError != null) {
// There was an error in the handler so make sure it's always captured...
fail("Stopping connector after error in the application's handler method: " + handlerError.getMessage(), handlerError);
}
try {
// First stop the task ...
logger.debug("Stopping the task and engine");
task.stop();
connectorCallback.ifPresent(ConnectorCallback::taskStopped);
// Always commit offsets that were captured from the source records we actually processed ...
commitOffsets(offsetWriter, commitTimeoutMs, task);
if (handlerError == null) {
// We stopped normally ...
succeed("Connector '" + connectorClassName + "' completed normally.");
}
} catch (Throwable t) {
fail("Error while trying to stop the task and commit the offsets", t);
}
}
} catch (Throwable t) {
fail("Error while trying to run connector class '" + connectorClassName + "'", t);
} finally {
// Close the offset storage and finally the connector ...
try {
offsetStore.stop();
} catch (Throwable t) {
fail("Error while trying to stop the offset store", t);
} finally {
try {
connector.stop();
connectorCallback.ifPresent(ConnectorCallback::connectorStopped);
} catch (Throwable t) {
fail("Error while trying to stop connector class '" + connectorClassName + "'", t);
}
}
}
} finally {
latch.countDown();
runningThread.set(null);
// after we've "shut down" the engine, fire the completion callback based on the results we collected
completionCallback.handle(completionResult.success(), completionResult.message(), completionResult.error());
}
}
}
Aggregations