Examples with RebalanceInProgressException org.apache.kafka.common.errors.RebalanceInProgressException used on opensource projects

Example 1 with RebalanceInProgressException

use of org.apache.kafka.common.errors.RebalanceInProgressException in project kafka by apache.

the class ConsumerCoordinator method sendOffsetCommitRequest.

/**
 * Commit offsets for the specified list of topics and partitions. This is a non-blocking call
 * which returns a request future that can be polled in the case of a synchronous commit or ignored in the
 * asynchronous case.
 *
 * NOTE: This is visible only for testing
 *
 * @param offsets The list of offsets per partition that should be committed.
 * @return A request future whose value indicates whether the commit was successful or not
 */
RequestFuture<Void> sendOffsetCommitRequest(final Map<TopicPartition, OffsetAndMetadata> offsets) {
    if (offsets.isEmpty())
        return RequestFuture.voidSuccess();
    Node coordinator = checkAndGetCoordinator();
    if (coordinator == null)
        return RequestFuture.coordinatorNotAvailable();
    // create the offset commit request
    Map<String, OffsetCommitRequestData.OffsetCommitRequestTopic> requestTopicDataMap = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
        TopicPartition topicPartition = entry.getKey();
        OffsetAndMetadata offsetAndMetadata = entry.getValue();
        if (offsetAndMetadata.offset() < 0) {
            return RequestFuture.failure(new IllegalArgumentException("Invalid offset: " + offsetAndMetadata.offset()));
        }
        OffsetCommitRequestData.OffsetCommitRequestTopic topic = requestTopicDataMap.getOrDefault(topicPartition.topic(), new OffsetCommitRequestData.OffsetCommitRequestTopic().setName(topicPartition.topic()));
        topic.partitions().add(new OffsetCommitRequestData.OffsetCommitRequestPartition().setPartitionIndex(topicPartition.partition()).setCommittedOffset(offsetAndMetadata.offset()).setCommittedLeaderEpoch(offsetAndMetadata.leaderEpoch().orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)).setCommittedMetadata(offsetAndMetadata.metadata()));
        requestTopicDataMap.put(topicPartition.topic(), topic);
    }
    final Generation generation;
    if (subscriptions.hasAutoAssignedPartitions()) {
        generation = generationIfStable();
        // the only thing we can do is fail the commit and let the user rejoin the group in poll().
        if (generation == null) {
            log.info("Failing OffsetCommit request since the consumer is not part of an active group");
            if (rebalanceInProgress()) {
                // CommitFailedException to indicate this is not a fatal error
                return RequestFuture.failure(new RebalanceInProgressException("Offset commit cannot be completed since the " + "consumer is undergoing a rebalance for auto partition assignment. You can try completing the rebalance " + "by calling poll() and then retry the operation."));
            } else {
                return RequestFuture.failure(new CommitFailedException("Offset commit cannot be completed since the " + "consumer is not part of an active group for auto partition assignment; it is likely that the consumer " + "was kicked out of the group."));
            }
        }
    } else {
        generation = Generation.NO_GENERATION;
    }
    OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(new OffsetCommitRequestData().setGroupId(this.rebalanceConfig.groupId).setGenerationId(generation.generationId).setMemberId(generation.memberId).setGroupInstanceId(rebalanceConfig.groupInstanceId.orElse(null)).setTopics(new ArrayList<>(requestTopicDataMap.values())));
    log.trace("Sending OffsetCommit request with {} to coordinator {}", offsets, coordinator);
    return client.send(coordinator, builder).compose(new OffsetCommitResponseHandler(offsets, generation));
}

Example 2 with RebalanceInProgressException

use of org.apache.kafka.common.errors.RebalanceInProgressException in project kafka by apache.

the class AbstractCoordinator method joinGroupIfNeeded.

/**
 * Joins the group without starting the heartbeat thread.
 *
 * If this function returns true, the state must always be in STABLE and heartbeat enabled.
 * If this function returns false, the state can be in one of the following:
 *  * UNJOINED: got error response but times out before being able to re-join, heartbeat disabled
 *  * PREPARING_REBALANCE: not yet received join-group response before timeout, heartbeat disabled
 *  * COMPLETING_REBALANCE: not yet received sync-group response before timeout, heartbeat enabled
 *
 * Visible for testing.
 *
 * @param timer Timer bounding how long this method can block
 * @throws KafkaException if the callback throws exception
 * @return true iff the operation succeeded
 */
boolean joinGroupIfNeeded(final Timer timer) {
    while (rejoinNeededOrPending()) {
        if (!ensureCoordinatorReady(timer)) {
            return false;
        }
        // still in progress.
        if (needsJoinPrepare) {
            // need to set the flag before calling onJoinPrepare since the user callback may throw
            // exception, in which case upon retry we should not retry onJoinPrepare either.
            needsJoinPrepare = false;
            // return false when onJoinPrepare is waiting for committing offset
            if (!onJoinPrepare(generation.generationId, generation.memberId)) {
                needsJoinPrepare = true;
                // should not initiateJoinGroup if needsJoinPrepare still is true
                return false;
            }
        }
        final RequestFuture<ByteBuffer> future = initiateJoinGroup();
        client.poll(future, timer);
        if (!future.isDone()) {
            // we ran out of time
            return false;
        }
        if (future.succeeded()) {
            Generation generationSnapshot;
            MemberState stateSnapshot;
            // See {@link PlaintextConsumerTest#testMaxPollIntervalMsDelayInAssignment}
            synchronized (AbstractCoordinator.this) {
                generationSnapshot = this.generation;
                stateSnapshot = this.state;
            }
            if (!hasGenerationReset(generationSnapshot) && stateSnapshot == MemberState.STABLE) {
                // Duplicate the buffer in case `onJoinComplete` does not complete and needs to be retried.
                ByteBuffer memberAssignment = future.value().duplicate();
                onJoinComplete(generationSnapshot.generationId, generationSnapshot.memberId, generationSnapshot.protocolName, memberAssignment);
                // Generally speaking we should always resetJoinGroupFuture once the future is done, but here
                // we can only reset the join group future after the completion callback returns. This ensures
                // that if the callback is woken up, we will retry it on the next joinGroupIfNeeded.
                // And because of that we should explicitly trigger resetJoinGroupFuture in other conditions below.
                resetJoinGroupFuture();
                needsJoinPrepare = true;
            } else {
                final String reason = String.format("rebalance failed since the generation/state was " + "modified by heartbeat thread to %s/%s before the rebalance callback triggered", generationSnapshot, stateSnapshot);
                resetStateAndRejoin(reason, true);
                resetJoinGroupFuture();
            }
        } else {
            final RuntimeException exception = future.exception();
            resetJoinGroupFuture();
            synchronized (AbstractCoordinator.this) {
                rejoinReason = String.format("rebalance failed due to '%s' (%s)", exception.getMessage(), exception.getClass().getSimpleName());
                rejoinNeeded = true;
            }
            if (exception instanceof UnknownMemberIdException || exception instanceof IllegalGenerationException || exception instanceof RebalanceInProgressException || exception instanceof MemberIdRequiredException)
                continue;
            else if (!future.isRetriable())
                throw exception;
            timer.sleep(rebalanceConfig.retryBackoffMs);
        }
    }
    return true;
}