Examples with OffsetCommitRequestData org.apache.kafka.common.message.OffsetCommitRequestData used on opensource projects

Example 1 with OffsetCommitRequestData

use of org.apache.kafka.common.message.OffsetCommitRequestData in project kafka by apache.

the class ConsumerCoordinator method sendOffsetCommitRequest.

/**
 * Commit offsets for the specified list of topics and partitions. This is a non-blocking call
 * which returns a request future that can be polled in the case of a synchronous commit or ignored in the
 * asynchronous case.
 *
 * NOTE: This is visible only for testing
 *
 * @param offsets The list of offsets per partition that should be committed.
 * @return A request future whose value indicates whether the commit was successful or not
 */
RequestFuture<Void> sendOffsetCommitRequest(final Map<TopicPartition, OffsetAndMetadata> offsets) {
    if (offsets.isEmpty())
        return RequestFuture.voidSuccess();
    Node coordinator = checkAndGetCoordinator();
    if (coordinator == null)
        return RequestFuture.coordinatorNotAvailable();
    // create the offset commit request
    Map<String, OffsetCommitRequestData.OffsetCommitRequestTopic> requestTopicDataMap = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
        TopicPartition topicPartition = entry.getKey();
        OffsetAndMetadata offsetAndMetadata = entry.getValue();
        if (offsetAndMetadata.offset() < 0) {
            return RequestFuture.failure(new IllegalArgumentException("Invalid offset: " + offsetAndMetadata.offset()));
        }
        OffsetCommitRequestData.OffsetCommitRequestTopic topic = requestTopicDataMap.getOrDefault(topicPartition.topic(), new OffsetCommitRequestData.OffsetCommitRequestTopic().setName(topicPartition.topic()));
        topic.partitions().add(new OffsetCommitRequestData.OffsetCommitRequestPartition().setPartitionIndex(topicPartition.partition()).setCommittedOffset(offsetAndMetadata.offset()).setCommittedLeaderEpoch(offsetAndMetadata.leaderEpoch().orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)).setCommittedMetadata(offsetAndMetadata.metadata()));
        requestTopicDataMap.put(topicPartition.topic(), topic);
    }
    final Generation generation;
    if (subscriptions.hasAutoAssignedPartitions()) {
        generation = generationIfStable();
        // the only thing we can do is fail the commit and let the user rejoin the group in poll().
        if (generation == null) {
            log.info("Failing OffsetCommit request since the consumer is not part of an active group");
            if (rebalanceInProgress()) {
                // CommitFailedException to indicate this is not a fatal error
                return RequestFuture.failure(new RebalanceInProgressException("Offset commit cannot be completed since the " + "consumer is undergoing a rebalance for auto partition assignment. You can try completing the rebalance " + "by calling poll() and then retry the operation."));
            } else {
                return RequestFuture.failure(new CommitFailedException("Offset commit cannot be completed since the " + "consumer is not part of an active group for auto partition assignment; it is likely that the consumer " + "was kicked out of the group."));
            }
        }
    } else {
        generation = Generation.NO_GENERATION;
    }
    OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(new OffsetCommitRequestData().setGroupId(this.rebalanceConfig.groupId).setGenerationId(generation.generationId).setMemberId(generation.memberId).setGroupInstanceId(rebalanceConfig.groupInstanceId.orElse(null)).setTopics(new ArrayList<>(requestTopicDataMap.values())));
    log.trace("Sending OffsetCommit request with {} to coordinator {}", offsets, coordinator);
    return client.send(coordinator, builder).compose(new OffsetCommitResponseHandler(offsets, generation));
}

Example 2 with OffsetCommitRequestData

use of org.apache.kafka.common.message.OffsetCommitRequestData in project kafka by apache.

the class ConsumerCoordinatorTest method testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead.

@Test
public void testCoordinatorUnknownInUnsentCallbacksAfterCoordinatorDead() {
    // When the coordinator is marked dead, all unsent or in-flight requests are cancelled
    // with a disconnect error. This test case ensures that the corresponding callbacks see
    // the coordinator as unknown which prevents additional retries to the same coordinator.
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    final AtomicBoolean asyncCallbackInvoked = new AtomicBoolean(false);
    OffsetCommitRequestData offsetCommitRequestData = new OffsetCommitRequestData().setGroupId(groupId).setTopics(Collections.singletonList(new OffsetCommitRequestData.OffsetCommitRequestTopic().setName("foo").setPartitions(Collections.singletonList(new OffsetCommitRequestData.OffsetCommitRequestPartition().setPartitionIndex(0).setCommittedLeaderEpoch(RecordBatch.NO_PARTITION_LEADER_EPOCH).setCommittedMetadata("").setCommittedOffset(13L).setCommitTimestamp(0)))));
    consumerClient.send(coordinator.checkAndGetCoordinator(), new OffsetCommitRequest.Builder(offsetCommitRequestData)).compose(new RequestFutureAdapter<ClientResponse, Object>() {

        @Override
        public void onSuccess(ClientResponse value, RequestFuture<Object> future) {
        }

        @Override
        public void onFailure(RuntimeException e, RequestFuture<Object> future) {
            assertTrue(e instanceof DisconnectException, "Unexpected exception type: " + e.getClass());
            assertTrue(coordinator.coordinatorUnknown());
            asyncCallbackInvoked.set(true);
        }
    });
    coordinator.markCoordinatorUnknown("test cause");
    consumerClient.pollNoWakeup();
    assertTrue(asyncCallbackInvoked.get());
}

Example 3 with OffsetCommitRequestData

use of org.apache.kafka.common.message.OffsetCommitRequestData in project kafka by apache.

the class OffsetCommitRequestTest method setUp.

@BeforeEach
public void setUp() {
    topics = Arrays.asList(new OffsetCommitRequestTopic().setName(topicOne).setPartitions(Collections.singletonList(new OffsetCommitRequestPartition().setPartitionIndex(partitionOne).setCommittedOffset(offset).setCommittedLeaderEpoch(leaderEpoch).setCommittedMetadata(metadata))), new OffsetCommitRequestTopic().setName(topicTwo).setPartitions(Collections.singletonList(new OffsetCommitRequestPartition().setPartitionIndex(partitionTwo).setCommittedOffset(offset).setCommittedLeaderEpoch(leaderEpoch).setCommittedMetadata(metadata))));
    data = new OffsetCommitRequestData().setGroupId(groupId).setTopics(topics);
}

Example 4 with OffsetCommitRequestData

use of org.apache.kafka.common.message.OffsetCommitRequestData in project kafka by apache.

the class AlterConsumerGroupOffsetsHandler method buildRequest.

@Override
public OffsetCommitRequest.Builder buildRequest(int coordinatorId, Set<CoordinatorKey> groupIds) {
    validateKeys(groupIds);
    Map<String, OffsetCommitRequestTopic> offsetData = new HashMap<>();
    offsets.forEach((topicPartition, offsetAndMetadata) -> {
        OffsetCommitRequestTopic topic = offsetData.computeIfAbsent(topicPartition.topic(), key -> new OffsetCommitRequestTopic().setName(topicPartition.topic()));
        topic.partitions().add(new OffsetCommitRequestPartition().setCommittedOffset(offsetAndMetadata.offset()).setCommittedLeaderEpoch(offsetAndMetadata.leaderEpoch().orElse(-1)).setCommittedMetadata(offsetAndMetadata.metadata()).setPartitionIndex(topicPartition.partition()));
    });
    OffsetCommitRequestData data = new OffsetCommitRequestData().setGroupId(groupId.idValue).setTopics(new ArrayList<>(offsetData.values()));
    return new OffsetCommitRequest.Builder(data);
}