Examples with RetryWithToleranceOperator org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator used on opensource projects

Example 6 with RetryWithToleranceOperator

use of org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator in project kafka by apache.

the class ErrorHandlingTaskTest method testCloseErrorReportersExceptionPropagation.

@Test
public void testCloseErrorReportersExceptionPropagation() {
    ErrorReporter reporterA = EasyMock.mock(ErrorReporter.class);
    ErrorReporter reporterB = EasyMock.mock(ErrorReporter.class);
    RetryWithToleranceOperator retryWithToleranceOperator = operator();
    retryWithToleranceOperator.metrics(errorHandlingMetrics);
    retryWithToleranceOperator.reporters(Arrays.asList(reporterA, reporterB));
    createSourceTask(initialState, retryWithToleranceOperator);
    expectClose();
    // Even though the reporters throw exceptions, they should both still be closed.
    reporterA.close();
    EasyMock.expectLastCall().andThrow(new RuntimeException());
    reporterB.close();
    EasyMock.expectLastCall().andThrow(new RuntimeException());
    PowerMock.replayAll();
    workerSourceTask.initialize(TASK_CONFIG);
    workerSourceTask.close();
    PowerMock.verifyAll();
}

Example 7 with RetryWithToleranceOperator

use of org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator in project kafka by apache.

the class Worker method buildWorkerTask.

private WorkerTask buildWorkerTask(ClusterConfigState configState, ConnectorConfig connConfig, ConnectorTaskId id, Task task, TaskStatus.Listener statusListener, TargetState initialState, Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter, ClassLoader loader) {
    ErrorHandlingMetrics errorHandlingMetrics = errorHandlingMetrics(id);
    final Class<? extends Connector> connectorClass = plugins.connectorClass(connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG));
    RetryWithToleranceOperator retryWithToleranceOperator = new RetryWithToleranceOperator(connConfig.errorRetryTimeout(), connConfig.errorMaxDelayInMillis(), connConfig.errorToleranceType(), Time.SYSTEM);
    retryWithToleranceOperator.metrics(errorHandlingMetrics);
    // Decide which type of worker task we need based on the type of task.
    if (task instanceof SourceTask) {
        SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connConfig.originalsStrings(), config.topicCreationEnable());
        retryWithToleranceOperator.reporters(sourceTaskReporters(id, sourceConfig, errorHandlingMetrics));
        TransformationChain<SourceRecord> transformationChain = new TransformationChain<>(sourceConfig.<SourceRecord>transformations(), retryWithToleranceOperator);
        log.info("Initializing: {}", transformationChain);
        CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        Map<String, Object> producerProps = producerConfigs(id, "connector-producer-" + id, config, sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
        KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(producerProps);
        TopicAdmin admin;
        Map<String, TopicCreationGroup> topicCreationGroups;
        if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) {
            Map<String, Object> adminProps = adminConfigs(id, "connector-adminclient-" + id, config, sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
            admin = new TopicAdmin(adminProps);
            topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig);
        } else {
            admin = null;
            topicCreationGroups = null;
        }
        // Note we pass the configState as it performs dynamic transformations under the covers
        return new WorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, producer, admin, topicCreationGroups, offsetReader, offsetWriter, config, configState, metrics, loader, time, retryWithToleranceOperator, herder.statusBackingStore(), executor);
    } else if (task instanceof SinkTask) {
        TransformationChain<SinkRecord> transformationChain = new TransformationChain<>(connConfig.<SinkRecord>transformations(), retryWithToleranceOperator);
        log.info("Initializing: {}", transformationChain);
        SinkConnectorConfig sinkConfig = new SinkConnectorConfig(plugins, connConfig.originalsStrings());
        retryWithToleranceOperator.reporters(sinkTaskReporters(id, sinkConfig, errorHandlingMetrics, connectorClass));
        WorkerErrantRecordReporter workerErrantRecordReporter = createWorkerErrantRecordReporter(sinkConfig, retryWithToleranceOperator, keyConverter, valueConverter, headerConverter);
        Map<String, Object> consumerProps = consumerConfigs(id, config, connConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
        KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps);
        return new WorkerSinkTask(id, (SinkTask) task, statusListener, initialState, config, configState, metrics, keyConverter, valueConverter, headerConverter, transformationChain, consumer, loader, time, retryWithToleranceOperator, workerErrantRecordReporter, herder.statusBackingStore());
    } else {
        log.error("Tasks must be a subclass of either SourceTask or SinkTask and current is {}", task);
        throw new ConnectException("Tasks must be a subclass of either SourceTask or SinkTask");
    }
}

Example 8 with RetryWithToleranceOperator

use of org.apache.kafka.connect.runtime.errors.RetryWithToleranceOperator in project kafka by apache.

the class WorkerTaskTest method stopBeforeStarting.

@Test
public void stopBeforeStarting() {
    ConnectorTaskId taskId = new ConnectorTaskId("foo", 0);
    WorkerTask workerTask = partialMockBuilder(WorkerTask.class).withConstructor(ConnectorTaskId.class, TaskStatus.Listener.class, TargetState.class, ClassLoader.class, ConnectMetrics.class, RetryWithToleranceOperator.class, Time.class, StatusBackingStore.class).withArgs(taskId, statusListener, TargetState.STARTED, loader, metrics, retryWithToleranceOperator, Time.SYSTEM, statusBackingStore).addMockedMethod("initialize").addMockedMethod("execute").addMockedMethod("close").createStrictMock();
    workerTask.initialize(TASK_CONFIG);
    EasyMock.expectLastCall();
    workerTask.close();
    EasyMock.expectLastCall();
    replay(workerTask);
    workerTask.initialize(TASK_CONFIG);
    workerTask.stop();
    workerTask.awaitStop(1000L);
    // now run should not do anything
    workerTask.run();
    verify(workerTask);
}