Search in sources :

Example 1 with EXACTLY_ONCE_V2

use of org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_V2 in project kafka by apache.

the class TaskExecutor method commitOffsetsOrTransaction.

/**
 * Caution: do not invoke this directly if it's possible a rebalance is occurring, as the commit will fail. If
 * this is a possibility, prefer the {@link #commitTasksAndMaybeUpdateCommittableOffsets} instead.
 *
 * @throws TaskMigratedException   if committing offsets failed due to CommitFailedException (non-EOS)
 * @throws TimeoutException        if committing offsets failed due to TimeoutException (non-EOS)
 * @throws TaskCorruptedException  if committing offsets failed due to TimeoutException (EOS)
 */
void commitOffsetsOrTransaction(final Map<Task, Map<TopicPartition, OffsetAndMetadata>> offsetsPerTask) {
    // avoid logging actual Task objects
    log.debug("Committing task offsets {}", offsetsPerTask.entrySet().stream().collect(Collectors.toMap(t -> t.getKey().id(), Entry::getValue)));
    final Set<TaskId> corruptedTasks = new HashSet<>();
    if (!offsetsPerTask.isEmpty()) {
        if (processingMode == EXACTLY_ONCE_ALPHA) {
            for (final Map.Entry<Task, Map<TopicPartition, OffsetAndMetadata>> taskToCommit : offsetsPerTask.entrySet()) {
                final Task task = taskToCommit.getKey();
                try {
                    tasks.streamsProducerForTask(task.id()).commitTransaction(taskToCommit.getValue(), tasks.mainConsumer().groupMetadata());
                    updateTaskCommitMetadata(taskToCommit.getValue());
                } catch (final TimeoutException timeoutException) {
                    log.error(String.format("Committing task %s failed.", task.id()), timeoutException);
                    corruptedTasks.add(task.id());
                }
            }
        } else {
            final Map<TopicPartition, OffsetAndMetadata> allOffsets = offsetsPerTask.values().stream().flatMap(e -> e.entrySet().stream()).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
            if (processingMode == EXACTLY_ONCE_V2) {
                try {
                    tasks.threadProducer().commitTransaction(allOffsets, tasks.mainConsumer().groupMetadata());
                    updateTaskCommitMetadata(allOffsets);
                } catch (final TimeoutException timeoutException) {
                    log.error(String.format("Committing task(s) %s failed.", offsetsPerTask.keySet().stream().map(t -> t.id().toString()).collect(Collectors.joining(", "))), timeoutException);
                    offsetsPerTask.keySet().forEach(task -> corruptedTasks.add(task.id()));
                }
            } else {
                try {
                    tasks.mainConsumer().commitSync(allOffsets);
                    updateTaskCommitMetadata(allOffsets);
                } catch (final CommitFailedException error) {
                    throw new TaskMigratedException("Consumer committing offsets failed, " + "indicating the corresponding thread is no longer part of the group", error);
                } catch (final TimeoutException timeoutException) {
                    log.error(String.format("Committing task(s) %s failed.", offsetsPerTask.keySet().stream().map(t -> t.id().toString()).collect(Collectors.joining(", "))), timeoutException);
                    throw timeoutException;
                } catch (final KafkaException error) {
                    throw new StreamsException("Error encountered committing offsets via consumer", error);
                }
            }
        }
        if (!corruptedTasks.isEmpty()) {
            throw new TaskCorruptedException(corruptedTasks);
        }
    }
}
Also used : TopicPartition(org.apache.kafka.common.TopicPartition) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Logger(org.slf4j.Logger) Time(org.apache.kafka.common.utils.Time) TaskId(org.apache.kafka.streams.processor.TaskId) Collection(java.util.Collection) KafkaException(org.apache.kafka.common.KafkaException) Set(java.util.Set) HashMap(java.util.HashMap) StreamsException(org.apache.kafka.streams.errors.StreamsException) ProcessingMode(org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode) Collectors(java.util.stream.Collectors) HashSet(java.util.HashSet) EXACTLY_ONCE_V2(org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_V2) EXACTLY_ONCE_ALPHA(org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_ALPHA) TaskMigratedException(org.apache.kafka.streams.errors.TaskMigratedException) LogContext(org.apache.kafka.common.utils.LogContext) Map(java.util.Map) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) Entry(java.util.Map.Entry) CommitFailedException(org.apache.kafka.clients.consumer.CommitFailedException) TaskCorruptedException(org.apache.kafka.streams.errors.TaskCorruptedException) TaskId(org.apache.kafka.streams.processor.TaskId) StreamsException(org.apache.kafka.streams.errors.StreamsException) Entry(java.util.Map.Entry) TaskCorruptedException(org.apache.kafka.streams.errors.TaskCorruptedException) TopicPartition(org.apache.kafka.common.TopicPartition) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) KafkaException(org.apache.kafka.common.KafkaException) HashMap(java.util.HashMap) Map(java.util.Map) CommitFailedException(org.apache.kafka.clients.consumer.CommitFailedException) HashSet(java.util.HashSet) TimeoutException(org.apache.kafka.common.errors.TimeoutException) TaskMigratedException(org.apache.kafka.streams.errors.TaskMigratedException)

Aggregations

Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Set (java.util.Set)1 Collectors (java.util.stream.Collectors)1 CommitFailedException (org.apache.kafka.clients.consumer.CommitFailedException)1 OffsetAndMetadata (org.apache.kafka.clients.consumer.OffsetAndMetadata)1 KafkaException (org.apache.kafka.common.KafkaException)1 TopicPartition (org.apache.kafka.common.TopicPartition)1 TimeoutException (org.apache.kafka.common.errors.TimeoutException)1 LogContext (org.apache.kafka.common.utils.LogContext)1 Time (org.apache.kafka.common.utils.Time)1 StreamsException (org.apache.kafka.streams.errors.StreamsException)1 TaskCorruptedException (org.apache.kafka.streams.errors.TaskCorruptedException)1 TaskMigratedException (org.apache.kafka.streams.errors.TaskMigratedException)1 ProcessingMode (org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode)1 EXACTLY_ONCE_ALPHA (org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_ALPHA)1 EXACTLY_ONCE_V2 (org.apache.kafka.streams.internals.StreamsConfigUtils.ProcessingMode.EXACTLY_ONCE_V2)1