Examples with ListOffsetsTopic org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic used on opensource projects

Example 1 with ListOffsetsTopic

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic in project kafka by apache.

the class ListOffsetsRequest method getErrorResponse.

@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
    short versionId = version();
    short errorCode = Errors.forException(e).code();
    List<ListOffsetsTopicResponse> responses = new ArrayList<>();
    for (ListOffsetsTopic topic : data.topics()) {
        ListOffsetsTopicResponse topicResponse = new ListOffsetsTopicResponse().setName(topic.name());
        List<ListOffsetsPartitionResponse> partitions = new ArrayList<>();
        for (ListOffsetsPartition partition : topic.partitions()) {
            ListOffsetsPartitionResponse partitionResponse = new ListOffsetsPartitionResponse().setErrorCode(errorCode).setPartitionIndex(partition.partitionIndex());
            if (versionId == 0) {
                partitionResponse.setOldStyleOffsets(Collections.emptyList());
            } else {
                partitionResponse.setOffset(ListOffsetsResponse.UNKNOWN_OFFSET).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP);
            }
            partitions.add(partitionResponse);
        }
        topicResponse.setPartitions(partitions);
        responses.add(topicResponse);
    }
    ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(throttleTimeMs).setTopics(responses);
    return new ListOffsetsResponse(responseData);
}

Example 2 with ListOffsetsTopic

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic in project kafka by apache.

the class ListOffsetsRequest method toListOffsetsTopics.

public static List<ListOffsetsTopic> toListOffsetsTopics(Map<TopicPartition, ListOffsetsPartition> timestampsToSearch) {
    Map<String, ListOffsetsTopic> topics = new HashMap<>();
    for (Map.Entry<TopicPartition, ListOffsetsPartition> entry : timestampsToSearch.entrySet()) {
        TopicPartition tp = entry.getKey();
        ListOffsetsTopic topic = topics.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
        topic.partitions().add(entry.getValue());
    }
    return new ArrayList<>(topics.values());
}

Example 3 with ListOffsetsTopic

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic in project kafka by apache.

the class KafkaAdminClient method getListOffsetsCalls.

// visible for benchmark
List<Call> getListOffsetsCalls(MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> context, Map<TopicPartition, OffsetSpec> topicPartitionOffsets, Map<TopicPartition, KafkaFutureImpl<ListOffsetsResultInfo>> futures) {
    MetadataResponse mr = context.response().orElseThrow(() -> new IllegalStateException("No Metadata response"));
    Cluster clusterSnapshot = mr.buildCluster();
    List<Call> calls = new ArrayList<>();
    // grouping topic partitions per leader
    Map<Node, Map<String, ListOffsetsTopic>> leaders = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetSpec> entry : topicPartitionOffsets.entrySet()) {
        OffsetSpec offsetSpec = entry.getValue();
        TopicPartition tp = entry.getKey();
        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
        long offsetQuery = getOffsetFromOffsetSpec(offsetSpec);
        // avoid sending listOffsets request for topics with errors
        if (!mr.errors().containsKey(tp.topic())) {
            Node node = clusterSnapshot.leaderFor(tp);
            if (node != null) {
                Map<String, ListOffsetsTopic> leadersOnNode = leaders.computeIfAbsent(node, k -> new HashMap<>());
                ListOffsetsTopic topic = leadersOnNode.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
                topic.partitions().add(new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offsetQuery));
            } else {
                future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
            }
        } else {
            future.completeExceptionally(mr.errors().get(tp.topic()).exception());
        }
    }
    for (final Map.Entry<Node, Map<String, ListOffsetsTopic>> entry : leaders.entrySet()) {
        final int brokerId = entry.getKey().id();
        calls.add(new Call("listOffsets on broker " + brokerId, context.deadline(), new ConstantNodeIdProvider(brokerId)) {

            final List<ListOffsetsTopic> partitionsToQuery = new ArrayList<>(entry.getValue().values());

            private boolean supportsMaxTimestamp = partitionsToQuery.stream().flatMap(t -> t.partitions().stream()).anyMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP);

            @Override
            ListOffsetsRequest.Builder createRequest(int timeoutMs) {
                return ListOffsetsRequest.Builder.forConsumer(true, context.options().isolationLevel(), supportsMaxTimestamp).setTargetTimes(partitionsToQuery);
            }

            @Override
            void handleResponse(AbstractResponse abstractResponse) {
                ListOffsetsResponse response = (ListOffsetsResponse) abstractResponse;
                Map<TopicPartition, OffsetSpec> retryTopicPartitionOffsets = new HashMap<>();
                for (ListOffsetsTopicResponse topic : response.topics()) {
                    for (ListOffsetsPartitionResponse partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        Errors error = Errors.forCode(partition.errorCode());
                        OffsetSpec offsetRequestSpec = topicPartitionOffsets.get(tp);
                        if (offsetRequestSpec == null) {
                            log.warn("Server response mentioned unknown topic partition {}", tp);
                        } else if (MetadataOperationContext.shouldRefreshMetadata(error)) {
                            retryTopicPartitionOffsets.put(tp, offsetRequestSpec);
                        } else if (error == Errors.NONE) {
                            Optional<Integer> leaderEpoch = (partition.leaderEpoch() == ListOffsetsResponse.UNKNOWN_EPOCH) ? Optional.empty() : Optional.of(partition.leaderEpoch());
                            future.complete(new ListOffsetsResultInfo(partition.offset(), partition.timestamp(), leaderEpoch));
                        } else {
                            future.completeExceptionally(error.exception());
                        }
                    }
                }
                if (retryTopicPartitionOffsets.isEmpty()) {
                    // The server should send back a response for every topic partition. But do a sanity check anyway.
                    for (ListOffsetsTopic topic : partitionsToQuery) {
                        for (ListOffsetsPartition partition : topic.partitions()) {
                            TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                            ApiException error = new ApiException("The response from broker " + brokerId + " did not contain a result for topic partition " + tp);
                            futures.get(tp).completeExceptionally(error);
                        }
                    }
                } else {
                    Set<String> retryTopics = retryTopicPartitionOffsets.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
                    MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> retryContext = new MetadataOperationContext<>(retryTopics, context.options(), context.deadline(), futures);
                    rescheduleMetadataTask(retryContext, () -> getListOffsetsCalls(retryContext, retryTopicPartitionOffsets, futures));
                }
            }

            @Override
            void handleFailure(Throwable throwable) {
                for (ListOffsetsTopic topic : entry.getValue().values()) {
                    for (ListOffsetsPartition partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        future.completeExceptionally(throwable);
                    }
                }
            }

            @Override
            boolean handleUnsupportedVersionException(UnsupportedVersionException exception) {
                if (supportsMaxTimestamp) {
                    supportsMaxTimestamp = false;
                    // fail any unsupported futures and remove partitions from the downgraded retry
                    Iterator<ListOffsetsTopic> topicIterator = partitionsToQuery.iterator();
                    while (topicIterator.hasNext()) {
                        ListOffsetsTopic topic = topicIterator.next();
                        Iterator<ListOffsetsPartition> partitionIterator = topic.partitions().iterator();
                        while (partitionIterator.hasNext()) {
                            ListOffsetsPartition partition = partitionIterator.next();
                            if (partition.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP) {
                                futures.get(new TopicPartition(topic.name(), partition.partitionIndex())).completeExceptionally(new UnsupportedVersionException("Broker " + brokerId + " does not support MAX_TIMESTAMP offset spec"));
                                partitionIterator.remove();
                            }
                        }
                        if (topic.partitions().isEmpty()) {
                            topicIterator.remove();
                        }
                    }
                    return !partitionsToQuery.isEmpty();
                }
                return false;
            }
        });
    }
    return calls;
}

Example 4 with ListOffsetsTopic

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic in project kafka by apache.

the class ListOffsetsRequestTest method testGetErrorResponseV0.

@Test
public void testGetErrorResponseV0() {
    List<ListOffsetsTopic> topics = Arrays.asList(new ListOffsetsTopic().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartition().setPartitionIndex(0))));
    ListOffsetsRequest request = ListOffsetsRequest.Builder.forConsumer(true, IsolationLevel.READ_UNCOMMITTED, false).setTargetTimes(topics).build((short) 0);
    ListOffsetsResponse response = (ListOffsetsResponse) request.getErrorResponse(0, Errors.NOT_LEADER_OR_FOLLOWER.exception());
    List<ListOffsetsTopicResponse> v = Collections.singletonList(new ListOffsetsTopicResponse().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartitionResponse().setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code()).setOldStyleOffsets(Collections.emptyList()).setPartitionIndex(0))));
    ListOffsetsResponseData data = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(v);
    ListOffsetsResponse expectedResponse = new ListOffsetsResponse(data);
    assertEquals(expectedResponse.data().topics(), response.data().topics());
    assertEquals(expectedResponse.throttleTimeMs(), response.throttleTimeMs());
}

Example 5 with ListOffsetsTopic

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic in project kafka by apache.

the class ListOffsetsRequestTest method testGetErrorResponse.

@Test
public void testGetErrorResponse() {
    for (short version = 1; version <= ApiKeys.LIST_OFFSETS.latestVersion(); version++) {
        List<ListOffsetsTopic> topics = Arrays.asList(new ListOffsetsTopic().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartition().setPartitionIndex(0))));
        ListOffsetsRequest request = ListOffsetsRequest.Builder.forConsumer(true, IsolationLevel.READ_COMMITTED, false).setTargetTimes(topics).build(version);
        ListOffsetsResponse response = (ListOffsetsResponse) request.getErrorResponse(0, Errors.NOT_LEADER_OR_FOLLOWER.exception());
        List<ListOffsetsTopicResponse> v = Collections.singletonList(new ListOffsetsTopicResponse().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartitionResponse().setErrorCode(Errors.NOT_LEADER_OR_FOLLOWER.code()).setLeaderEpoch(ListOffsetsResponse.UNKNOWN_EPOCH).setOffset(ListOffsetsResponse.UNKNOWN_OFFSET).setPartitionIndex(0).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP))));
        ListOffsetsResponseData data = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(v);
        ListOffsetsResponse expectedResponse = new ListOffsetsResponse(data);
        assertEquals(expectedResponse.data().topics(), response.data().topics());
        assertEquals(expectedResponse.throttleTimeMs(), response.throttleTimeMs());
    }
}