Examples with ListOffsetsPartition org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition used on opensource projects

Example 1 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class ListOffsetsRequest method getErrorResponse.

@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
    short versionId = version();
    short errorCode = Errors.forException(e).code();
    List<ListOffsetsTopicResponse> responses = new ArrayList<>();
    for (ListOffsetsTopic topic : data.topics()) {
        ListOffsetsTopicResponse topicResponse = new ListOffsetsTopicResponse().setName(topic.name());
        List<ListOffsetsPartitionResponse> partitions = new ArrayList<>();
        for (ListOffsetsPartition partition : topic.partitions()) {
            ListOffsetsPartitionResponse partitionResponse = new ListOffsetsPartitionResponse().setErrorCode(errorCode).setPartitionIndex(partition.partitionIndex());
            if (versionId == 0) {
                partitionResponse.setOldStyleOffsets(Collections.emptyList());
            } else {
                partitionResponse.setOffset(ListOffsetsResponse.UNKNOWN_OFFSET).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP);
            }
            partitions.add(partitionResponse);
        }
        topicResponse.setPartitions(partitions);
        responses.add(topicResponse);
    }
    ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(throttleTimeMs).setTopics(responses);
    return new ListOffsetsResponse(responseData);
}

Example 2 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class ListOffsetsRequest method toListOffsetsTopics.

public static List<ListOffsetsTopic> toListOffsetsTopics(Map<TopicPartition, ListOffsetsPartition> timestampsToSearch) {
    Map<String, ListOffsetsTopic> topics = new HashMap<>();
    for (Map.Entry<TopicPartition, ListOffsetsPartition> entry : timestampsToSearch.entrySet()) {
        TopicPartition tp = entry.getKey();
        ListOffsetsTopic topic = topics.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
        topic.partitions().add(entry.getValue());
    }
    return new ArrayList<>(topics.values());
}

Example 3 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class KafkaAdminClient method getListOffsetsCalls.

// visible for benchmark
List<Call> getListOffsetsCalls(MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> context, Map<TopicPartition, OffsetSpec> topicPartitionOffsets, Map<TopicPartition, KafkaFutureImpl<ListOffsetsResultInfo>> futures) {
    MetadataResponse mr = context.response().orElseThrow(() -> new IllegalStateException("No Metadata response"));
    Cluster clusterSnapshot = mr.buildCluster();
    List<Call> calls = new ArrayList<>();
    // grouping topic partitions per leader
    Map<Node, Map<String, ListOffsetsTopic>> leaders = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetSpec> entry : topicPartitionOffsets.entrySet()) {
        OffsetSpec offsetSpec = entry.getValue();
        TopicPartition tp = entry.getKey();
        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
        long offsetQuery = getOffsetFromOffsetSpec(offsetSpec);
        // avoid sending listOffsets request for topics with errors
        if (!mr.errors().containsKey(tp.topic())) {
            Node node = clusterSnapshot.leaderFor(tp);
            if (node != null) {
                Map<String, ListOffsetsTopic> leadersOnNode = leaders.computeIfAbsent(node, k -> new HashMap<>());
                ListOffsetsTopic topic = leadersOnNode.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
                topic.partitions().add(new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offsetQuery));
            } else {
                future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
            }
        } else {
            future.completeExceptionally(mr.errors().get(tp.topic()).exception());
        }
    }
    for (final Map.Entry<Node, Map<String, ListOffsetsTopic>> entry : leaders.entrySet()) {
        final int brokerId = entry.getKey().id();
        calls.add(new Call("listOffsets on broker " + brokerId, context.deadline(), new ConstantNodeIdProvider(brokerId)) {

            final List<ListOffsetsTopic> partitionsToQuery = new ArrayList<>(entry.getValue().values());

            private boolean supportsMaxTimestamp = partitionsToQuery.stream().flatMap(t -> t.partitions().stream()).anyMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP);

            @Override
            ListOffsetsRequest.Builder createRequest(int timeoutMs) {
                return ListOffsetsRequest.Builder.forConsumer(true, context.options().isolationLevel(), supportsMaxTimestamp).setTargetTimes(partitionsToQuery);
            }

            @Override
            void handleResponse(AbstractResponse abstractResponse) {
                ListOffsetsResponse response = (ListOffsetsResponse) abstractResponse;
                Map<TopicPartition, OffsetSpec> retryTopicPartitionOffsets = new HashMap<>();
                for (ListOffsetsTopicResponse topic : response.topics()) {
                    for (ListOffsetsPartitionResponse partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        Errors error = Errors.forCode(partition.errorCode());
                        OffsetSpec offsetRequestSpec = topicPartitionOffsets.get(tp);
                        if (offsetRequestSpec == null) {
                            log.warn("Server response mentioned unknown topic partition {}", tp);
                        } else if (MetadataOperationContext.shouldRefreshMetadata(error)) {
                            retryTopicPartitionOffsets.put(tp, offsetRequestSpec);
                        } else if (error == Errors.NONE) {
                            Optional<Integer> leaderEpoch = (partition.leaderEpoch() == ListOffsetsResponse.UNKNOWN_EPOCH) ? Optional.empty() : Optional.of(partition.leaderEpoch());
                            future.complete(new ListOffsetsResultInfo(partition.offset(), partition.timestamp(), leaderEpoch));
                        } else {
                            future.completeExceptionally(error.exception());
                        }
                    }
                }
                if (retryTopicPartitionOffsets.isEmpty()) {
                    // The server should send back a response for every topic partition. But do a sanity check anyway.
                    for (ListOffsetsTopic topic : partitionsToQuery) {
                        for (ListOffsetsPartition partition : topic.partitions()) {
                            TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                            ApiException error = new ApiException("The response from broker " + brokerId + " did not contain a result for topic partition " + tp);
                            futures.get(tp).completeExceptionally(error);
                        }
                    }
                } else {
                    Set<String> retryTopics = retryTopicPartitionOffsets.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
                    MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> retryContext = new MetadataOperationContext<>(retryTopics, context.options(), context.deadline(), futures);
                    rescheduleMetadataTask(retryContext, () -> getListOffsetsCalls(retryContext, retryTopicPartitionOffsets, futures));
                }
            }

            @Override
            void handleFailure(Throwable throwable) {
                for (ListOffsetsTopic topic : entry.getValue().values()) {
                    for (ListOffsetsPartition partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        future.completeExceptionally(throwable);
                    }
                }
            }

            @Override
            boolean handleUnsupportedVersionException(UnsupportedVersionException exception) {
                if (supportsMaxTimestamp) {
                    supportsMaxTimestamp = false;
                    // fail any unsupported futures and remove partitions from the downgraded retry
                    Iterator<ListOffsetsTopic> topicIterator = partitionsToQuery.iterator();
                    while (topicIterator.hasNext()) {
                        ListOffsetsTopic topic = topicIterator.next();
                        Iterator<ListOffsetsPartition> partitionIterator = topic.partitions().iterator();
                        while (partitionIterator.hasNext()) {
                            ListOffsetsPartition partition = partitionIterator.next();
                            if (partition.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP) {
                                futures.get(new TopicPartition(topic.name(), partition.partitionIndex())).completeExceptionally(new UnsupportedVersionException("Broker " + brokerId + " does not support MAX_TIMESTAMP offset spec"));
                                partitionIterator.remove();
                            }
                        }
                        if (topic.partitions().isEmpty()) {
                            topicIterator.remove();
                        }
                    }
                    return !partitionsToQuery.isEmpty();
                }
                return false;
            }
        });
    }
    return calls;
}

Example 4 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class Fetcher method resetOffsetsAsync.

private void resetOffsetsAsync(Map<TopicPartition, Long> partitionResetTimestamps) {
    Map<Node, Map<TopicPartition, ListOffsetsPartition>> timestampsToSearchByNode = groupListOffsetRequests(partitionResetTimestamps, new HashSet<>());
    for (Map.Entry<Node, Map<TopicPartition, ListOffsetsPartition>> entry : timestampsToSearchByNode.entrySet()) {
        Node node = entry.getKey();
        final Map<TopicPartition, ListOffsetsPartition> resetTimestamps = entry.getValue();
        subscriptions.setNextAllowedRetry(resetTimestamps.keySet(), time.milliseconds() + requestTimeoutMs);
        RequestFuture<ListOffsetResult> future = sendListOffsetRequest(node, resetTimestamps, false);
        future.addListener(new RequestFutureListener<ListOffsetResult>() {

            @Override
            public void onSuccess(ListOffsetResult result) {
                if (!result.partitionsToRetry.isEmpty()) {
                    subscriptions.requestFailed(result.partitionsToRetry, time.milliseconds() + retryBackoffMs);
                    metadata.requestUpdate();
                }
                for (Map.Entry<TopicPartition, ListOffsetData> fetchedOffset : result.fetchedOffsets.entrySet()) {
                    TopicPartition partition = fetchedOffset.getKey();
                    ListOffsetData offsetData = fetchedOffset.getValue();
                    ListOffsetsPartition requestedReset = resetTimestamps.get(partition);
                    resetOffsetIfNeeded(partition, timestampToOffsetResetStrategy(requestedReset.timestamp()), offsetData);
                }
            }

            @Override
            public void onFailure(RuntimeException e) {
                subscriptions.requestFailed(resetTimestamps.keySet(), time.milliseconds() + retryBackoffMs);
                metadata.requestUpdate();
                if (!(e instanceof RetriableException) && !cachedListOffsetsException.compareAndSet(null, e))
                    log.error("Discarding error in ListOffsetResponse because another error is pending", e);
            }
        });
    }
}

Example 5 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class Fetcher method groupListOffsetRequests.

/**
 * Groups timestamps to search by node for topic partitions in `timestampsToSearch` that have
 * leaders available. Topic partitions from `timestampsToSearch` that do not have their leader
 * available are added to `partitionsToRetry`
 * @param timestampsToSearch The mapping from partitions ot the target timestamps
 * @param partitionsToRetry A set of topic partitions that will be extended with partitions
 *                          that need metadata update or re-connect to the leader.
 */
private Map<Node, Map<TopicPartition, ListOffsetsPartition>> groupListOffsetRequests(Map<TopicPartition, Long> timestampsToSearch, Set<TopicPartition> partitionsToRetry) {
    final Map<TopicPartition, ListOffsetsPartition> partitionDataMap = new HashMap<>();
    for (Map.Entry<TopicPartition, Long> entry : timestampsToSearch.entrySet()) {
        TopicPartition tp = entry.getKey();
        Long offset = entry.getValue();
        Metadata.LeaderAndEpoch leaderAndEpoch = metadata.currentLeader(tp);
        if (!leaderAndEpoch.leader.isPresent()) {
            log.debug("Leader for partition {} is unknown for fetching offset {}", tp, offset);
            metadata.requestUpdate();
            partitionsToRetry.add(tp);
        } else {
            Node leader = leaderAndEpoch.leader.get();
            if (client.isUnavailable(leader)) {
                client.maybeThrowAuthFailure(leader);
                // The connection has failed and we need to await the backoff period before we can
                // try again. No need to request a metadata update since the disconnect will have
                // done so already.
                log.debug("Leader {} for partition {} is unavailable for fetching offset until reconnect backoff expires", leader, tp);
                partitionsToRetry.add(tp);
            } else {
                int currentLeaderEpoch = leaderAndEpoch.epoch.orElse(ListOffsetsResponse.UNKNOWN_EPOCH);
                partitionDataMap.put(tp, new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offset).setCurrentLeaderEpoch(currentLeaderEpoch));
            }
        }
    }
    return regroupPartitionMapByNode(partitionDataMap);
}