Examples with OffsetCommitPolicy io.debezium.embedded.spi.OffsetCommitPolicy used on opensource projects

Example 1 with OffsetCommitPolicy

use of io.debezium.embedded.spi.OffsetCommitPolicy in project debezium by debezium.

the class OffsetCommitPolicyTest method shouldAlwaysCommit.

@Test
public void shouldAlwaysCommit() {
    OffsetCommitPolicy policy = OffsetCommitPolicy.always();
    assertThat(policy.performCommit(0, Duration.ofNanos(0))).isTrue();
    assertThat(policy.performCommit(10000, Duration.ofDays(1000))).isTrue();
}

Example 2 with OffsetCommitPolicy

use of io.debezium.embedded.spi.OffsetCommitPolicy in project debezium by debezium.

the class OffsetCommitPolicyTest method shouldCombineOnePolicyWithNull.

@Test
public void shouldCombineOnePolicyWithNull() {
    AtomicBoolean commit = new AtomicBoolean(false);
    OffsetCommitPolicy policy1 = (num, time) -> commit.get();
    assertThat(policy1.and(null)).isSameAs(policy1);
    assertThat(policy1.or(null)).isSameAs(policy1);
}

Example 3 with OffsetCommitPolicy

use of io.debezium.embedded.spi.OffsetCommitPolicy in project debezium by debezium.

the class OffsetCommitPolicyTest method shouldCombineTwoPolicies.

@Test
public void shouldCombineTwoPolicies() {
    AtomicBoolean commitFirst = new AtomicBoolean(false);
    AtomicBoolean commitSecond = new AtomicBoolean(false);
    OffsetCommitPolicy policy1 = (num, time) -> commitFirst.get();
    OffsetCommitPolicy policy2 = (num, time) -> commitSecond.get();
    OffsetCommitPolicy both1 = policy1.and(policy2);
    OffsetCommitPolicy both2 = policy2.and(policy1);
    OffsetCommitPolicy either1 = policy1.or(policy2);
    OffsetCommitPolicy either2 = policy2.or(policy1);
    assertThat(both1.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(both2.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(either1.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(either2.performCommit(0, Duration.ofNanos(0))).isFalse();
    commitFirst.set(true);
    assertThat(both1.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(both2.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(either1.performCommit(0, Duration.ofNanos(0))).isTrue();
    assertThat(either2.performCommit(0, Duration.ofNanos(0))).isTrue();
    commitSecond.set(true);
    assertThat(both1.performCommit(0, Duration.ofNanos(0))).isTrue();
    assertThat(both2.performCommit(0, Duration.ofNanos(0))).isTrue();
    assertThat(either1.performCommit(0, Duration.ofNanos(0))).isTrue();
    assertThat(either2.performCommit(0, Duration.ofNanos(0))).isTrue();
    commitFirst.set(false);
    assertThat(both1.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(both2.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(either1.performCommit(0, Duration.ofNanos(0))).isTrue();
    assertThat(either2.performCommit(0, Duration.ofNanos(0))).isTrue();
    commitSecond.set(false);
    assertThat(both1.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(both2.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(either1.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(either2.performCommit(0, Duration.ofNanos(0))).isFalse();
}

Example 4 with OffsetCommitPolicy

use of io.debezium.embedded.spi.OffsetCommitPolicy in project debezium by debezium.

the class EmbeddedEngine method run.

/**
 * Run this embedded connector and deliver database changes to the registered {@link Consumer}. This method blocks until
 * the connector is stopped.
 * <p>
 * First, the method checks to see if this instance is currently {@link #run() running}, and if so immediately returns.
 * <p>
 * If the configuration is valid, this method starts the connector and starts polling the connector for change events.
 * All messages are delivered in batches to the {@link Consumer} registered with this embedded connector. The batch size,
 * polling
 * frequency, and other parameters are controlled via configuration settings. This continues until this connector is
 * {@link #stop() stopped}.
 * <p>
 * Note that there are two ways to stop a connector running on a thread: calling {@link #stop()} from another thread, or
 * interrupting the thread (e.g., via {@link ExecutorService#shutdownNow()}).
 * <p>
 * This method can be called repeatedly as needed.
 */
@Override
public void run() {
    if (runningThread.compareAndSet(null, Thread.currentThread())) {
        final String engineName = config.getString(ENGINE_NAME);
        final String connectorClassName = config.getString(CONNECTOR_CLASS);
        final Optional<ConnectorCallback> connectorCallback = Optional.ofNullable(this.connectorCallback);
        // Only one thread can be in this part of the method at a time ...
        latch.countUp();
        try {
            if (!config.validateAndRecord(CONNECTOR_FIELDS, logger::error)) {
                fail("Failed to start connector with invalid configuration (see logs for actual errors)");
                return;
            }
            // Instantiate the connector ...
            SourceConnector connector = null;
            try {
                @SuppressWarnings("unchecked") Class<? extends SourceConnector> connectorClass = (Class<SourceConnector>) classLoader.loadClass(connectorClassName);
                connector = connectorClass.newInstance();
            } catch (Throwable t) {
                fail("Unable to instantiate connector class '" + connectorClassName + "'", t);
                return;
            }
            // Instantiate the offset store ...
            final String offsetStoreClassName = config.getString(OFFSET_STORAGE);
            OffsetBackingStore offsetStore = null;
            try {
                @SuppressWarnings("unchecked") Class<? extends OffsetBackingStore> offsetStoreClass = (Class<OffsetBackingStore>) classLoader.loadClass(offsetStoreClassName);
                offsetStore = offsetStoreClass.newInstance();
            } catch (Throwable t) {
                fail("Unable to instantiate OffsetBackingStore class '" + offsetStoreClassName + "'", t);
                return;
            }
            // Initialize the offset store ...
            try {
                offsetStore.configure(workerConfig);
                offsetStore.start();
            } catch (Throwable t) {
                fail("Unable to configure and start the '" + offsetStoreClassName + "' offset backing store", t);
                return;
            }
            // Set up the offset commit policy ...
            if (offsetCommitPolicy == null) {
                offsetCommitPolicy = config.getInstance(EmbeddedEngine.OFFSET_COMMIT_POLICY, OffsetCommitPolicy.class, config);
            }
            // Initialize the connector using a context that does NOT respond to requests to reconfigure tasks ...
            ConnectorContext context = new ConnectorContext() {

                @Override
                public void requestTaskReconfiguration() {
                // Do nothing ...
                }

                @Override
                public void raiseError(Exception e) {
                    fail(e.getMessage(), e);
                }
            };
            connector.initialize(context);
            OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetStore, engineName, keyConverter, valueConverter);
            OffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetStore, engineName, keyConverter, valueConverter);
            long commitTimeoutMs = config.getLong(OFFSET_COMMIT_TIMEOUT_MS);
            try {
                // Start the connector with the given properties and get the task configurations ...
                connector.start(config.asMap());
                connectorCallback.ifPresent(ConnectorCallback::connectorStarted);
                List<Map<String, String>> taskConfigs = connector.taskConfigs(1);
                Class<? extends Task> taskClass = connector.taskClass();
                SourceTask task = null;
                try {
                    task = (SourceTask) taskClass.newInstance();
                } catch (IllegalAccessException | InstantiationException t) {
                    fail("Unable to instantiate connector's task class '" + taskClass.getName() + "'", t);
                    return;
                }
                try {
                    SourceTaskContext taskContext = () -> offsetReader;
                    task.initialize(taskContext);
                    task.start(taskConfigs.get(0));
                    connectorCallback.ifPresent(ConnectorCallback::taskStarted);
                } catch (Throwable t) {
                    // Mask the passwords ...
                    Configuration config = Configuration.from(taskConfigs.get(0)).withMaskedPasswords();
                    String msg = "Unable to initialize and start connector's task class '" + taskClass.getName() + "' with config: " + config;
                    fail(msg, t);
                    return;
                }
                recordsSinceLastCommit = 0;
                Throwable handlerError = null;
                try {
                    timeOfLastCommitMillis = clock.currentTimeInMillis();
                    boolean keepProcessing = true;
                    List<SourceRecord> changeRecords = null;
                    while (runningThread.get() != null && handlerError == null && keepProcessing) {
                        try {
                            try {
                                logger.debug("Embedded engine is polling task for records on thread " + runningThread.get());
                                // blocks until there are values ...
                                changeRecords = task.poll();
                                logger.debug("Embedded engine returned from polling task for records");
                            } catch (InterruptedException e) {
                                // Interrupted while polling ...
                                logger.debug("Embedded engine interrupted on thread " + runningThread.get() + " while polling the task for records");
                                Thread.interrupted();
                                break;
                            }
                            try {
                                if (changeRecords != null && !changeRecords.isEmpty()) {
                                    logger.debug("Received {} records from the task", changeRecords.size());
                                    // First forward the records to the connector's consumer ...
                                    for (SourceRecord record : changeRecords) {
                                        try {
                                            consumer.accept(record);
                                            task.commitRecord(record);
                                        } catch (StopConnectorException e) {
                                            keepProcessing = false;
                                            // Stop processing any more but first record the offset for this record's
                                            // partition
                                            offsetWriter.offset(record.sourcePartition(), record.sourceOffset());
                                            recordsSinceLastCommit += 1;
                                            break;
                                        } catch (Throwable t) {
                                            handlerError = t;
                                            break;
                                        }
                                        // Record the offset for this record's partition
                                        offsetWriter.offset(record.sourcePartition(), record.sourceOffset());
                                        recordsSinceLastCommit += 1;
                                    }
                                    // Flush the offsets to storage if necessary ...
                                    maybeFlush(offsetWriter, offsetCommitPolicy, commitTimeoutMs, task);
                                } else {
                                    logger.debug("Received no records from the task");
                                }
                            } catch (Throwable t) {
                                // There was some sort of unexpected exception, so we should stop work
                                if (handlerError == null) {
                                    // make sure we capture the error first so that we can report it later
                                    handlerError = t;
                                }
                                break;
                            }
                        } finally {
                            // then try to commit the offsets, since we record them only after the records were handled
                            // by the consumer ...
                            maybeFlush(offsetWriter, offsetCommitPolicy, commitTimeoutMs, task);
                        }
                    }
                } finally {
                    if (handlerError != null) {
                        // There was an error in the handler so make sure it's always captured...
                        fail("Stopping connector after error in the application's handler method: " + handlerError.getMessage(), handlerError);
                    }
                    try {
                        // First stop the task ...
                        logger.debug("Stopping the task and engine");
                        task.stop();
                        connectorCallback.ifPresent(ConnectorCallback::taskStopped);
                        // Always commit offsets that were captured from the source records we actually processed ...
                        commitOffsets(offsetWriter, commitTimeoutMs, task);
                        if (handlerError == null) {
                            // We stopped normally ...
                            succeed("Connector '" + connectorClassName + "' completed normally.");
                        }
                    } catch (Throwable t) {
                        fail("Error while trying to stop the task and commit the offsets", t);
                    }
                }
            } catch (Throwable t) {
                fail("Error while trying to run connector class '" + connectorClassName + "'", t);
            } finally {
                // Close the offset storage and finally the connector ...
                try {
                    offsetStore.stop();
                } catch (Throwable t) {
                    fail("Error while trying to stop the offset store", t);
                } finally {
                    try {
                        connector.stop();
                        connectorCallback.ifPresent(ConnectorCallback::connectorStopped);
                    } catch (Throwable t) {
                        fail("Error while trying to stop connector class '" + connectorClassName + "'", t);
                    }
                }
            }
        } finally {
            latch.countDown();
            runningThread.set(null);
            // after we've "shut down" the engine, fire the completion callback based on the results we collected
            completionCallback.handle(completionResult.success(), completionResult.message(), completionResult.error());
        }
    }
}

Example 5 with OffsetCommitPolicy

use of io.debezium.embedded.spi.OffsetCommitPolicy in project debezium by debezium.

the class OffsetCommitPolicyTest method shouldCommitPeriodically.

@Test
public void shouldCommitPeriodically() {
    // 10 hours
    OffsetCommitPolicy policy = OffsetCommitPolicy.periodic(Configuration.create().with(EmbeddedEngine.OFFSET_FLUSH_INTERVAL_MS, 10 * 60 * 60 * 1000).build());
    assertThat(policy.performCommit(0, Duration.ofNanos(0))).isFalse();
    assertThat(policy.performCommit(10000, Duration.ofHours(9))).isFalse();
    assertThat(policy.performCommit(0, Duration.ofHours(10))).isTrue();
}