Examples with OffsetForEpochResult org.apache.kafka.clients.consumer.internals.OffsetsForLeaderEpochClient.OffsetForEpochResult used on opensource projects

Example 1 with OffsetForEpochResult

use of org.apache.kafka.clients.consumer.internals.OffsetsForLeaderEpochClient.OffsetForEpochResult in project kafka by apache.

the class Fetcher method validateOffsetsAsync.

/**
 * For each partition which needs validation, make an asynchronous request to get the end-offsets for the partition
 * with the epoch less than or equal to the epoch the partition last saw.
 *
 * Requests are grouped by Node for efficiency.
 */
private void validateOffsetsAsync(Map<TopicPartition, FetchPosition> partitionsToValidate) {
    final Map<Node, Map<TopicPartition, FetchPosition>> regrouped = regroupFetchPositionsByLeader(partitionsToValidate);
    long nextResetTimeMs = time.milliseconds() + requestTimeoutMs;
    regrouped.forEach((node, fetchPositions) -> {
        if (node.isEmpty()) {
            metadata.requestUpdate();
            return;
        }
        NodeApiVersions nodeApiVersions = apiVersions.get(node.idString());
        if (nodeApiVersions == null) {
            client.tryConnect(node);
            return;
        }
        if (!hasUsableOffsetForLeaderEpochVersion(nodeApiVersions)) {
            log.debug("Skipping validation of fetch offsets for partitions {} since the broker does not " + "support the required protocol version (introduced in Kafka 2.3)", fetchPositions.keySet());
            for (TopicPartition partition : fetchPositions.keySet()) {
                subscriptions.completeValidation(partition);
            }
            return;
        }
        subscriptions.setNextAllowedRetry(fetchPositions.keySet(), nextResetTimeMs);
        RequestFuture<OffsetForEpochResult> future = offsetsForLeaderEpochClient.sendAsyncRequest(node, fetchPositions);
        future.addListener(new RequestFutureListener<OffsetForEpochResult>() {

            @Override
            public void onSuccess(OffsetForEpochResult offsetsResult) {
                List<SubscriptionState.LogTruncation> truncations = new ArrayList<>();
                if (!offsetsResult.partitionsToRetry().isEmpty()) {
                    subscriptions.setNextAllowedRetry(offsetsResult.partitionsToRetry(), time.milliseconds() + retryBackoffMs);
                    metadata.requestUpdate();
                }
                // For each OffsetsForLeader response, check if the end-offset is lower than our current offset
                // for the partition. If so, it means we have experienced log truncation and need to reposition
                // that partition's offset.
                // 
                // In addition, check whether the returned offset and epoch are valid. If not, then we should reset
                // its offset if reset policy is configured, or throw out of range exception.
                offsetsResult.endOffsets().forEach((topicPartition, respEndOffset) -> {
                    FetchPosition requestPosition = fetchPositions.get(topicPartition);
                    Optional<SubscriptionState.LogTruncation> truncationOpt = subscriptions.maybeCompleteValidation(topicPartition, requestPosition, respEndOffset);
                    truncationOpt.ifPresent(truncations::add);
                });
                if (!truncations.isEmpty()) {
                    maybeSetOffsetForLeaderException(buildLogTruncationException(truncations));
                }
            }

            @Override
            public void onFailure(RuntimeException e) {
                subscriptions.requestFailed(fetchPositions.keySet(), time.milliseconds() + retryBackoffMs);
                metadata.requestUpdate();
                if (!(e instanceof RetriableException)) {
                    maybeSetOffsetForLeaderException(e);
                }
            }
        });
    });
}