Examples with CloseableOffsetStorageReader org.apache.kafka.connect.storage.CloseableOffsetStorageReader used on opensource projects

Example 1 with CloseableOffsetStorageReader

use of org.apache.kafka.connect.storage.CloseableOffsetStorageReader in project kafka by apache.

the class Worker method buildWorkerTask.

private WorkerTask buildWorkerTask(ClusterConfigState configState, ConnectorConfig connConfig, ConnectorTaskId id, Task task, TaskStatus.Listener statusListener, TargetState initialState, Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter, ClassLoader loader) {
    ErrorHandlingMetrics errorHandlingMetrics = errorHandlingMetrics(id);
    final Class<? extends Connector> connectorClass = plugins.connectorClass(connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG));
    RetryWithToleranceOperator retryWithToleranceOperator = new RetryWithToleranceOperator(connConfig.errorRetryTimeout(), connConfig.errorMaxDelayInMillis(), connConfig.errorToleranceType(), Time.SYSTEM);
    retryWithToleranceOperator.metrics(errorHandlingMetrics);
    // Decide which type of worker task we need based on the type of task.
    if (task instanceof SourceTask) {
        SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connConfig.originalsStrings(), config.topicCreationEnable());
        retryWithToleranceOperator.reporters(sourceTaskReporters(id, sourceConfig, errorHandlingMetrics));
        TransformationChain<SourceRecord> transformationChain = new TransformationChain<>(sourceConfig.<SourceRecord>transformations(), retryWithToleranceOperator);
        log.info("Initializing: {}", transformationChain);
        CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        Map<String, Object> producerProps = producerConfigs(id, "connector-producer-" + id, config, sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
        KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(producerProps);
        TopicAdmin admin;
        Map<String, TopicCreationGroup> topicCreationGroups;
        if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) {
            Map<String, Object> adminProps = adminConfigs(id, "connector-adminclient-" + id, config, sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
            admin = new TopicAdmin(adminProps);
            topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig);
        } else {
            admin = null;
            topicCreationGroups = null;
        }
        // Note we pass the configState as it performs dynamic transformations under the covers
        return new WorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, producer, admin, topicCreationGroups, offsetReader, offsetWriter, config, configState, metrics, loader, time, retryWithToleranceOperator, herder.statusBackingStore(), executor);
    } else if (task instanceof SinkTask) {
        TransformationChain<SinkRecord> transformationChain = new TransformationChain<>(connConfig.<SinkRecord>transformations(), retryWithToleranceOperator);
        log.info("Initializing: {}", transformationChain);
        SinkConnectorConfig sinkConfig = new SinkConnectorConfig(plugins, connConfig.originalsStrings());
        retryWithToleranceOperator.reporters(sinkTaskReporters(id, sinkConfig, errorHandlingMetrics, connectorClass));
        WorkerErrantRecordReporter workerErrantRecordReporter = createWorkerErrantRecordReporter(sinkConfig, retryWithToleranceOperator, keyConverter, valueConverter, headerConverter);
        Map<String, Object> consumerProps = consumerConfigs(id, config, connConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
        KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps);
        return new WorkerSinkTask(id, (SinkTask) task, statusListener, initialState, config, configState, metrics, keyConverter, valueConverter, headerConverter, transformationChain, consumer, loader, time, retryWithToleranceOperator, workerErrantRecordReporter, herder.statusBackingStore());
    } else {
        log.error("Tasks must be a subclass of either SourceTask or SinkTask and current is {}", task);
        throw new ConnectException("Tasks must be a subclass of either SourceTask or SinkTask");
    }
}