Examples with CommitFailedException org.apache.kafka.clients.consumer.CommitFailedException used on opensource projects

Example 1 with CommitFailedException

use of org.apache.kafka.clients.consumer.CommitFailedException in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinator method sendOffsetCommitRequest.

/**
 * Commit offsets for the specified list of topics and partitions. This is a non-blocking call
 * which returns a request future that can be polled in the case of a synchronous commit or ignored in the
 * asynchronous case.
 *
 * @param offsets The list of offsets per partition that should be committed.
 * @return A request future whose value indicates whether the commit was successful or not
 */
private RequestFuture<Void> sendOffsetCommitRequest(final Map<TopicPartition, OffsetAndMetadata> offsets) {
    if (offsets.isEmpty())
        return RequestFuture.voidSuccess();
    Node coordinator = checkAndGetCoordinator();
    if (coordinator == null)
        return RequestFuture.coordinatorNotAvailable();
    // create the offset commit request
    Map<TopicPartition, OffsetCommitRequest.PartitionData> offsetData = new HashMap<>(offsets.size());
    for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
        OffsetAndMetadata offsetAndMetadata = entry.getValue();
        if (offsetAndMetadata.offset() < 0) {
            return RequestFuture.failure(new IllegalArgumentException("Invalid offset: " + offsetAndMetadata.offset()));
        }
        offsetData.put(entry.getKey(), new OffsetCommitRequest.PartitionData(offsetAndMetadata.offset(), offsetAndMetadata.metadata()));
    }
    final Generation generation;
    if (subscriptions.partitionsAutoAssigned())
        generation = generation();
    else
        generation = Generation.NO_GENERATION;
    // the only thing we can do is fail the commit and let the user rejoin the group in poll()
    if (generation == null)
        return RequestFuture.failure(new CommitFailedException());
    OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(this.groupId, offsetData).setGenerationId(generation.generationId).setMemberId(generation.memberId).setRetentionTime(OffsetCommitRequest.DEFAULT_RETENTION_TIME);
    log.trace("Sending OffsetCommit request with {} to coordinator {}", offsets, coordinator);
    return client.send(coordinator, builder).compose(new OffsetCommitResponseHandler(offsets));
}

Example 2 with CommitFailedException

use of org.apache.kafka.clients.consumer.CommitFailedException in project kafka by apache.

the class ConsumerCoordinator method sendOffsetCommitRequest.

/**
 * Commit offsets for the specified list of topics and partitions. This is a non-blocking call
 * which returns a request future that can be polled in the case of a synchronous commit or ignored in the
 * asynchronous case.
 *
 * NOTE: This is visible only for testing
 *
 * @param offsets The list of offsets per partition that should be committed.
 * @return A request future whose value indicates whether the commit was successful or not
 */
RequestFuture<Void> sendOffsetCommitRequest(final Map<TopicPartition, OffsetAndMetadata> offsets) {
    if (offsets.isEmpty())
        return RequestFuture.voidSuccess();
    Node coordinator = checkAndGetCoordinator();
    if (coordinator == null)
        return RequestFuture.coordinatorNotAvailable();
    // create the offset commit request
    Map<String, OffsetCommitRequestData.OffsetCommitRequestTopic> requestTopicDataMap = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
        TopicPartition topicPartition = entry.getKey();
        OffsetAndMetadata offsetAndMetadata = entry.getValue();
        if (offsetAndMetadata.offset() < 0) {
            return RequestFuture.failure(new IllegalArgumentException("Invalid offset: " + offsetAndMetadata.offset()));
        }
        OffsetCommitRequestData.OffsetCommitRequestTopic topic = requestTopicDataMap.getOrDefault(topicPartition.topic(), new OffsetCommitRequestData.OffsetCommitRequestTopic().setName(topicPartition.topic()));
        topic.partitions().add(new OffsetCommitRequestData.OffsetCommitRequestPartition().setPartitionIndex(topicPartition.partition()).setCommittedOffset(offsetAndMetadata.offset()).setCommittedLeaderEpoch(offsetAndMetadata.leaderEpoch().orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)).setCommittedMetadata(offsetAndMetadata.metadata()));
        requestTopicDataMap.put(topicPartition.topic(), topic);
    }
    final Generation generation;
    if (subscriptions.hasAutoAssignedPartitions()) {
        generation = generationIfStable();
        // the only thing we can do is fail the commit and let the user rejoin the group in poll().
        if (generation == null) {
            log.info("Failing OffsetCommit request since the consumer is not part of an active group");
            if (rebalanceInProgress()) {
                // CommitFailedException to indicate this is not a fatal error
                return RequestFuture.failure(new RebalanceInProgressException("Offset commit cannot be completed since the " + "consumer is undergoing a rebalance for auto partition assignment. You can try completing the rebalance " + "by calling poll() and then retry the operation."));
            } else {
                return RequestFuture.failure(new CommitFailedException("Offset commit cannot be completed since the " + "consumer is not part of an active group for auto partition assignment; it is likely that the consumer " + "was kicked out of the group."));
            }
        }
    } else {
        generation = Generation.NO_GENERATION;
    }
    OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(new OffsetCommitRequestData().setGroupId(this.rebalanceConfig.groupId).setGenerationId(generation.generationId).setMemberId(generation.memberId).setGroupInstanceId(rebalanceConfig.groupInstanceId.orElse(null)).setTopics(new ArrayList<>(requestTopicDataMap.values())));
    log.trace("Sending OffsetCommit request with {} to coordinator {}", offsets, coordinator);
    return client.send(coordinator, builder).compose(new OffsetCommitResponseHandler(offsets, generation));
}

Example 3 with CommitFailedException

use of org.apache.kafka.clients.consumer.CommitFailedException in project kafka by apache.

the class TaskManagerTest method shouldThrowTaskMigratedExceptionOnCommitFailed.

@Test
public void shouldThrowTaskMigratedExceptionOnCommitFailed() {
    final StateMachineTask task01 = new StateMachineTask(taskId01, taskId01Partitions, true);
    final Map<TopicPartition, OffsetAndMetadata> offsets = singletonMap(t1p0, new OffsetAndMetadata(0L, null));
    task01.setCommittableOffsetsAndMetadata(offsets);
    task01.setCommitNeeded();
    taskManager.addTask(task01);
    consumer.commitSync(offsets);
    expectLastCall().andThrow(new CommitFailedException());
    replay(consumer);
    final TaskMigratedException thrown = assertThrows(TaskMigratedException.class, () -> taskManager.commitAll());
    assertThat(thrown.getCause(), instanceOf(CommitFailedException.class));
    assertThat(thrown.getMessage(), equalTo("Consumer committing offsets failed, indicating the corresponding thread is no longer part of the group;" + " it means all tasks belonging to this thread should be migrated."));
    assertThat(task01.state(), is(Task.State.CREATED));
}

Example 4 with CommitFailedException

use of org.apache.kafka.clients.consumer.CommitFailedException in project kafka by apache.

the class TaskExecutor method commitOffsetsOrTransaction.

/**
 * Caution: do not invoke this directly if it's possible a rebalance is occurring, as the commit will fail. If
 * this is a possibility, prefer the {@link #commitTasksAndMaybeUpdateCommittableOffsets} instead.
 *
 * @throws TaskMigratedException   if committing offsets failed due to CommitFailedException (non-EOS)
 * @throws TimeoutException        if committing offsets failed due to TimeoutException (non-EOS)
 * @throws TaskCorruptedException  if committing offsets failed due to TimeoutException (EOS)
 */
void commitOffsetsOrTransaction(final Map<Task, Map<TopicPartition, OffsetAndMetadata>> offsetsPerTask) {
    // avoid logging actual Task objects
    log.debug("Committing task offsets {}", offsetsPerTask.entrySet().stream().collect(Collectors.toMap(t -> t.getKey().id(), Entry::getValue)));
    final Set<TaskId> corruptedTasks = new HashSet<>();
    if (!offsetsPerTask.isEmpty()) {
        if (processingMode == EXACTLY_ONCE_ALPHA) {
            for (final Map.Entry<Task, Map<TopicPartition, OffsetAndMetadata>> taskToCommit : offsetsPerTask.entrySet()) {
                final Task task = taskToCommit.getKey();
                try {
                    tasks.streamsProducerForTask(task.id()).commitTransaction(taskToCommit.getValue(), tasks.mainConsumer().groupMetadata());
                    updateTaskCommitMetadata(taskToCommit.getValue());
                } catch (final TimeoutException timeoutException) {
                    log.error(String.format("Committing task %s failed.", task.id()), timeoutException);
                    corruptedTasks.add(task.id());
                }
            }
        } else {
            final Map<TopicPartition, OffsetAndMetadata> allOffsets = offsetsPerTask.values().stream().flatMap(e -> e.entrySet().stream()).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
            if (processingMode == EXACTLY_ONCE_V2) {
                try {
                    tasks.threadProducer().commitTransaction(allOffsets, tasks.mainConsumer().groupMetadata());
                    updateTaskCommitMetadata(allOffsets);
                } catch (final TimeoutException timeoutException) {
                    log.error(String.format("Committing task(s) %s failed.", offsetsPerTask.keySet().stream().map(t -> t.id().toString()).collect(Collectors.joining(", "))), timeoutException);
                    offsetsPerTask.keySet().forEach(task -> corruptedTasks.add(task.id()));
                }
            } else {
                try {
                    tasks.mainConsumer().commitSync(allOffsets);
                    updateTaskCommitMetadata(allOffsets);
                } catch (final CommitFailedException error) {
                    throw new TaskMigratedException("Consumer committing offsets failed, " + "indicating the corresponding thread is no longer part of the group", error);
                } catch (final TimeoutException timeoutException) {
                    log.error(String.format("Committing task(s) %s failed.", offsetsPerTask.keySet().stream().map(t -> t.id().toString()).collect(Collectors.joining(", "))), timeoutException);
                    throw timeoutException;
                } catch (final KafkaException error) {
                    throw new StreamsException("Error encountered committing offsets via consumer", error);
                }
            }
        }
        if (!corruptedTasks.isEmpty()) {
            throw new TaskCorruptedException(corruptedTasks);
        }
    }
}

Example 5 with CommitFailedException

use of org.apache.kafka.clients.consumer.CommitFailedException in project kafka by apache.

the class StreamTask method commitOffsets.

/**
     * commit consumed offsets if needed
     */
@Override
public void commitOffsets() {
    if (commitOffsetNeeded) {
        Map<TopicPartition, OffsetAndMetadata> consumedOffsetsAndMetadata = new HashMap<>(consumedOffsets.size());
        for (Map.Entry<TopicPartition, Long> entry : consumedOffsets.entrySet()) {
            TopicPartition partition = entry.getKey();
            long offset = entry.getValue() + 1;
            consumedOffsetsAndMetadata.put(partition, new OffsetAndMetadata(offset));
            stateMgr.putOffsetLimit(partition, offset);
        }
        try {
            consumer.commitSync(consumedOffsetsAndMetadata);
        } catch (final CommitFailedException cfe) {
            log.warn("{} Failed offset commits: {} ", logPrefix, consumedOffsetsAndMetadata);
            throw cfe;
        }
        commitOffsetNeeded = false;
    }
    commitRequested = false;
}