Examples with WorkerErrantRecordReporter org.apache.kafka.connect.runtime.errors.WorkerErrantRecordReporter used on opensource projects

Example 1 with WorkerErrantRecordReporter

use of org.apache.kafka.connect.runtime.errors.WorkerErrantRecordReporter in project kafka by apache.

the class WorkerSinkTask method commitOffsets.

private void commitOffsets(long now, boolean closing, Collection<TopicPartition> topicPartitions) {
    log.trace("Committing offsets for partitions {}", topicPartitions);
    if (workerErrantRecordReporter != null) {
        log.trace("Awaiting reported errors to be completed");
        workerErrantRecordReporter.awaitFutures(topicPartitions);
        log.trace("Completed reported errors");
    }
    Map<TopicPartition, OffsetAndMetadata> offsetsToCommit = currentOffsets.entrySet().stream().filter(e -> topicPartitions.contains(e.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    if (offsetsToCommit.isEmpty())
        return;
    committing = true;
    commitSeqno += 1;
    commitStarted = now;
    sinkTaskMetricsGroup.recordOffsetSequenceNumber(commitSeqno);
    Map<TopicPartition, OffsetAndMetadata> lastCommittedOffsetsForPartitions = this.lastCommittedOffsets.entrySet().stream().filter(e -> offsetsToCommit.containsKey(e.getKey())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    final Map<TopicPartition, OffsetAndMetadata> taskProvidedOffsets;
    try {
        log.trace("{} Calling task.preCommit with current offsets: {}", this, offsetsToCommit);
        taskProvidedOffsets = task.preCommit(new HashMap<>(offsetsToCommit));
    } catch (Throwable t) {
        if (closing) {
            log.warn("{} Offset commit failed during close", this);
        } else {
            log.error("{} Offset commit failed, rewinding to last committed offsets", this, t);
            for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : lastCommittedOffsetsForPartitions.entrySet()) {
                log.debug("{} Rewinding topic partition {} to offset {}", this, entry.getKey(), entry.getValue().offset());
                consumer.seek(entry.getKey(), entry.getValue().offset());
            }
            currentOffsets.putAll(lastCommittedOffsetsForPartitions);
        }
        onCommitCompleted(t, commitSeqno, null);
        return;
    } finally {
        if (closing) {
            log.trace("{} Closing the task before committing the offsets: {}", this, offsetsToCommit);
            task.close(topicPartitions);
        }
    }
    if (taskProvidedOffsets.isEmpty()) {
        log.debug("{} Skipping offset commit, task opted-out by returning no offsets from preCommit", this);
        onCommitCompleted(null, commitSeqno, null);
        return;
    }
    Collection<TopicPartition> allAssignedTopicPartitions = consumer.assignment();
    final Map<TopicPartition, OffsetAndMetadata> committableOffsets = new HashMap<>(lastCommittedOffsetsForPartitions);
    for (Map.Entry<TopicPartition, OffsetAndMetadata> taskProvidedOffsetEntry : taskProvidedOffsets.entrySet()) {
        final TopicPartition partition = taskProvidedOffsetEntry.getKey();
        final OffsetAndMetadata taskProvidedOffset = taskProvidedOffsetEntry.getValue();
        if (committableOffsets.containsKey(partition)) {
            long taskOffset = taskProvidedOffset.offset();
            long currentOffset = offsetsToCommit.get(partition).offset();
            if (taskOffset <= currentOffset) {
                committableOffsets.put(partition, taskProvidedOffset);
            } else {
                log.warn("{} Ignoring invalid task provided offset {}/{} -- not yet consumed, taskOffset={} currentOffset={}", this, partition, taskProvidedOffset, taskOffset, currentOffset);
            }
        } else if (!allAssignedTopicPartitions.contains(partition)) {
            log.warn("{} Ignoring invalid task provided offset {}/{} -- partition not assigned, assignment={}", this, partition, taskProvidedOffset, allAssignedTopicPartitions);
        } else {
            log.debug("{} Ignoring task provided offset {}/{} -- partition not requested, requested={}", this, partition, taskProvidedOffset, committableOffsets.keySet());
        }
    }
    if (committableOffsets.equals(lastCommittedOffsetsForPartitions)) {
        log.debug("{} Skipping offset commit, no change since last commit", this);
        onCommitCompleted(null, commitSeqno, null);
        return;
    }
    doCommit(committableOffsets, closing, commitSeqno);
}

Example 2 with WorkerErrantRecordReporter

use of org.apache.kafka.connect.runtime.errors.WorkerErrantRecordReporter in project kafka by apache.

the class Worker method buildWorkerTask.

private WorkerTask buildWorkerTask(ClusterConfigState configState, ConnectorConfig connConfig, ConnectorTaskId id, Task task, TaskStatus.Listener statusListener, TargetState initialState, Converter keyConverter, Converter valueConverter, HeaderConverter headerConverter, ClassLoader loader) {
    ErrorHandlingMetrics errorHandlingMetrics = errorHandlingMetrics(id);
    final Class<? extends Connector> connectorClass = plugins.connectorClass(connConfig.getString(ConnectorConfig.CONNECTOR_CLASS_CONFIG));
    RetryWithToleranceOperator retryWithToleranceOperator = new RetryWithToleranceOperator(connConfig.errorRetryTimeout(), connConfig.errorMaxDelayInMillis(), connConfig.errorToleranceType(), Time.SYSTEM);
    retryWithToleranceOperator.metrics(errorHandlingMetrics);
    // Decide which type of worker task we need based on the type of task.
    if (task instanceof SourceTask) {
        SourceConnectorConfig sourceConfig = new SourceConnectorConfig(plugins, connConfig.originalsStrings(), config.topicCreationEnable());
        retryWithToleranceOperator.reporters(sourceTaskReporters(id, sourceConfig, errorHandlingMetrics));
        TransformationChain<SourceRecord> transformationChain = new TransformationChain<>(sourceConfig.<SourceRecord>transformations(), retryWithToleranceOperator);
        log.info("Initializing: {}", transformationChain);
        CloseableOffsetStorageReader offsetReader = new OffsetStorageReaderImpl(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        OffsetStorageWriter offsetWriter = new OffsetStorageWriter(offsetBackingStore, id.connector(), internalKeyConverter, internalValueConverter);
        Map<String, Object> producerProps = producerConfigs(id, "connector-producer-" + id, config, sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
        KafkaProducer<byte[], byte[]> producer = new KafkaProducer<>(producerProps);
        TopicAdmin admin;
        Map<String, TopicCreationGroup> topicCreationGroups;
        if (config.topicCreationEnable() && sourceConfig.usesTopicCreation()) {
            Map<String, Object> adminProps = adminConfigs(id, "connector-adminclient-" + id, config, sourceConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
            admin = new TopicAdmin(adminProps);
            topicCreationGroups = TopicCreationGroup.configuredGroups(sourceConfig);
        } else {
            admin = null;
            topicCreationGroups = null;
        }
        // Note we pass the configState as it performs dynamic transformations under the covers
        return new WorkerSourceTask(id, (SourceTask) task, statusListener, initialState, keyConverter, valueConverter, headerConverter, transformationChain, producer, admin, topicCreationGroups, offsetReader, offsetWriter, config, configState, metrics, loader, time, retryWithToleranceOperator, herder.statusBackingStore(), executor);
    } else if (task instanceof SinkTask) {
        TransformationChain<SinkRecord> transformationChain = new TransformationChain<>(connConfig.<SinkRecord>transformations(), retryWithToleranceOperator);
        log.info("Initializing: {}", transformationChain);
        SinkConnectorConfig sinkConfig = new SinkConnectorConfig(plugins, connConfig.originalsStrings());
        retryWithToleranceOperator.reporters(sinkTaskReporters(id, sinkConfig, errorHandlingMetrics, connectorClass));
        WorkerErrantRecordReporter workerErrantRecordReporter = createWorkerErrantRecordReporter(sinkConfig, retryWithToleranceOperator, keyConverter, valueConverter, headerConverter);
        Map<String, Object> consumerProps = consumerConfigs(id, config, connConfig, connectorClass, connectorClientConfigOverridePolicy, kafkaClusterId);
        KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(consumerProps);
        return new WorkerSinkTask(id, (SinkTask) task, statusListener, initialState, config, configState, metrics, keyConverter, valueConverter, headerConverter, transformationChain, consumer, loader, time, retryWithToleranceOperator, workerErrantRecordReporter, herder.statusBackingStore());
    } else {
        log.error("Tasks must be a subclass of either SourceTask or SinkTask and current is {}", task);
        throw new ConnectException("Tasks must be a subclass of either SourceTask or SinkTask");
    }
}