Examples with MetadataResponse org.apache.kafka.common.requests.MetadataResponse used on opensource projects

Example 11 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaAdminClient method getListOffsetsCalls.

// visible for benchmark
List<Call> getListOffsetsCalls(MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> context, Map<TopicPartition, OffsetSpec> topicPartitionOffsets, Map<TopicPartition, KafkaFutureImpl<ListOffsetsResultInfo>> futures) {
    MetadataResponse mr = context.response().orElseThrow(() -> new IllegalStateException("No Metadata response"));
    Cluster clusterSnapshot = mr.buildCluster();
    List<Call> calls = new ArrayList<>();
    // grouping topic partitions per leader
    Map<Node, Map<String, ListOffsetsTopic>> leaders = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetSpec> entry : topicPartitionOffsets.entrySet()) {
        OffsetSpec offsetSpec = entry.getValue();
        TopicPartition tp = entry.getKey();
        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
        long offsetQuery = getOffsetFromOffsetSpec(offsetSpec);
        // avoid sending listOffsets request for topics with errors
        if (!mr.errors().containsKey(tp.topic())) {
            Node node = clusterSnapshot.leaderFor(tp);
            if (node != null) {
                Map<String, ListOffsetsTopic> leadersOnNode = leaders.computeIfAbsent(node, k -> new HashMap<>());
                ListOffsetsTopic topic = leadersOnNode.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
                topic.partitions().add(new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offsetQuery));
            } else {
                future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
            }
        } else {
            future.completeExceptionally(mr.errors().get(tp.topic()).exception());
        }
    }
    for (final Map.Entry<Node, Map<String, ListOffsetsTopic>> entry : leaders.entrySet()) {
        final int brokerId = entry.getKey().id();
        calls.add(new Call("listOffsets on broker " + brokerId, context.deadline(), new ConstantNodeIdProvider(brokerId)) {

            final List<ListOffsetsTopic> partitionsToQuery = new ArrayList<>(entry.getValue().values());

            private boolean supportsMaxTimestamp = partitionsToQuery.stream().flatMap(t -> t.partitions().stream()).anyMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP);

            @Override
            ListOffsetsRequest.Builder createRequest(int timeoutMs) {
                return ListOffsetsRequest.Builder.forConsumer(true, context.options().isolationLevel(), supportsMaxTimestamp).setTargetTimes(partitionsToQuery);
            }

            @Override
            void handleResponse(AbstractResponse abstractResponse) {
                ListOffsetsResponse response = (ListOffsetsResponse) abstractResponse;
                Map<TopicPartition, OffsetSpec> retryTopicPartitionOffsets = new HashMap<>();
                for (ListOffsetsTopicResponse topic : response.topics()) {
                    for (ListOffsetsPartitionResponse partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        Errors error = Errors.forCode(partition.errorCode());
                        OffsetSpec offsetRequestSpec = topicPartitionOffsets.get(tp);
                        if (offsetRequestSpec == null) {
                            log.warn("Server response mentioned unknown topic partition {}", tp);
                        } else if (MetadataOperationContext.shouldRefreshMetadata(error)) {
                            retryTopicPartitionOffsets.put(tp, offsetRequestSpec);
                        } else if (error == Errors.NONE) {
                            Optional<Integer> leaderEpoch = (partition.leaderEpoch() == ListOffsetsResponse.UNKNOWN_EPOCH) ? Optional.empty() : Optional.of(partition.leaderEpoch());
                            future.complete(new ListOffsetsResultInfo(partition.offset(), partition.timestamp(), leaderEpoch));
                        } else {
                            future.completeExceptionally(error.exception());
                        }
                    }
                }
                if (retryTopicPartitionOffsets.isEmpty()) {
                    // The server should send back a response for every topic partition. But do a sanity check anyway.
                    for (ListOffsetsTopic topic : partitionsToQuery) {
                        for (ListOffsetsPartition partition : topic.partitions()) {
                            TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                            ApiException error = new ApiException("The response from broker " + brokerId + " did not contain a result for topic partition " + tp);
                            futures.get(tp).completeExceptionally(error);
                        }
                    }
                } else {
                    Set<String> retryTopics = retryTopicPartitionOffsets.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
                    MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> retryContext = new MetadataOperationContext<>(retryTopics, context.options(), context.deadline(), futures);
                    rescheduleMetadataTask(retryContext, () -> getListOffsetsCalls(retryContext, retryTopicPartitionOffsets, futures));
                }
            }

            @Override
            void handleFailure(Throwable throwable) {
                for (ListOffsetsTopic topic : entry.getValue().values()) {
                    for (ListOffsetsPartition partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        future.completeExceptionally(throwable);
                    }
                }
            }

            @Override
            boolean handleUnsupportedVersionException(UnsupportedVersionException exception) {
                if (supportsMaxTimestamp) {
                    supportsMaxTimestamp = false;
                    // fail any unsupported futures and remove partitions from the downgraded retry
                    Iterator<ListOffsetsTopic> topicIterator = partitionsToQuery.iterator();
                    while (topicIterator.hasNext()) {
                        ListOffsetsTopic topic = topicIterator.next();
                        Iterator<ListOffsetsPartition> partitionIterator = topic.partitions().iterator();
                        while (partitionIterator.hasNext()) {
                            ListOffsetsPartition partition = partitionIterator.next();
                            if (partition.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP) {
                                futures.get(new TopicPartition(topic.name(), partition.partitionIndex())).completeExceptionally(new UnsupportedVersionException("Broker " + brokerId + " does not support MAX_TIMESTAMP offset spec"));
                                partitionIterator.remove();
                            }
                        }
                        if (topic.partitions().isEmpty()) {
                            topicIterator.remove();
                        }
                    }
                    return !partitionsToQuery.isEmpty();
                }
                return false;
            }
        });
    }
    return calls;
}

Example 12 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaAdminClient method handleDescribeTopicsByIds.

private Map<Uuid, KafkaFuture<TopicDescription>> handleDescribeTopicsByIds(Collection<Uuid> topicIds, DescribeTopicsOptions options) {
    final Map<Uuid, KafkaFutureImpl<TopicDescription>> topicFutures = new HashMap<>(topicIds.size());
    final List<Uuid> topicIdsList = new ArrayList<>();
    for (Uuid topicId : topicIds) {
        if (topicIdIsUnrepresentable(topicId)) {
            KafkaFutureImpl<TopicDescription> future = new KafkaFutureImpl<>();
            future.completeExceptionally(new InvalidTopicException("The given topic id '" + topicId + "' cannot be represented in a request."));
            topicFutures.put(topicId, future);
        } else if (!topicFutures.containsKey(topicId)) {
            topicFutures.put(topicId, new KafkaFutureImpl<>());
            topicIdsList.add(topicId);
        }
    }
    final long now = time.milliseconds();
    Call call = new Call("describeTopicsWithIds", calcDeadlineMs(now, options.timeoutMs()), new LeastLoadedNodeProvider()) {

        @Override
        MetadataRequest.Builder createRequest(int timeoutMs) {
            return new MetadataRequest.Builder(new MetadataRequestData().setTopics(convertTopicIdsToMetadataRequestTopic(topicIdsList)).setAllowAutoTopicCreation(false).setIncludeTopicAuthorizedOperations(options.includeAuthorizedOperations()));
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            MetadataResponse response = (MetadataResponse) abstractResponse;
            // Handle server responses for particular topics.
            Cluster cluster = response.buildCluster();
            Map<Uuid, Errors> errors = response.errorsByTopicId();
            for (Map.Entry<Uuid, KafkaFutureImpl<TopicDescription>> entry : topicFutures.entrySet()) {
                Uuid topicId = entry.getKey();
                KafkaFutureImpl<TopicDescription> future = entry.getValue();
                String topicName = cluster.topicName(topicId);
                if (topicName == null) {
                    future.completeExceptionally(new InvalidTopicException("TopicId " + topicId + " not found."));
                    continue;
                }
                Errors topicError = errors.get(topicId);
                if (topicError != null) {
                    future.completeExceptionally(topicError.exception());
                    continue;
                }
                Integer authorizedOperations = response.topicAuthorizedOperations(topicName).get();
                TopicDescription topicDescription = getTopicDescriptionFromCluster(cluster, topicName, topicId, authorizedOperations);
                future.complete(topicDescription);
            }
        }

        @Override
        void handleFailure(Throwable throwable) {
            completeAllExceptionally(topicFutures.values(), throwable);
        }
    };
    if (!topicIdsList.isEmpty()) {
        runnable.call(call, now);
    }
    return new HashMap<>(topicFutures);
}

Example 13 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class KafkaAdminClient method handleDescribeTopicsByNames.

private Map<String, KafkaFuture<TopicDescription>> handleDescribeTopicsByNames(final Collection<String> topicNames, DescribeTopicsOptions options) {
    final Map<String, KafkaFutureImpl<TopicDescription>> topicFutures = new HashMap<>(topicNames.size());
    final ArrayList<String> topicNamesList = new ArrayList<>();
    for (String topicName : topicNames) {
        if (topicNameIsUnrepresentable(topicName)) {
            KafkaFutureImpl<TopicDescription> future = new KafkaFutureImpl<>();
            future.completeExceptionally(new InvalidTopicException("The given topic name '" + topicName + "' cannot be represented in a request."));
            topicFutures.put(topicName, future);
        } else if (!topicFutures.containsKey(topicName)) {
            topicFutures.put(topicName, new KafkaFutureImpl<>());
            topicNamesList.add(topicName);
        }
    }
    final long now = time.milliseconds();
    Call call = new Call("describeTopics", calcDeadlineMs(now, options.timeoutMs()), new LeastLoadedNodeProvider()) {

        private boolean supportsDisablingTopicCreation = true;

        @Override
        MetadataRequest.Builder createRequest(int timeoutMs) {
            if (supportsDisablingTopicCreation)
                return new MetadataRequest.Builder(new MetadataRequestData().setTopics(convertToMetadataRequestTopic(topicNamesList)).setAllowAutoTopicCreation(false).setIncludeTopicAuthorizedOperations(options.includeAuthorizedOperations()));
            else
                return MetadataRequest.Builder.allTopics();
        }

        @Override
        void handleResponse(AbstractResponse abstractResponse) {
            MetadataResponse response = (MetadataResponse) abstractResponse;
            // Handle server responses for particular topics.
            Cluster cluster = response.buildCluster();
            Map<String, Errors> errors = response.errors();
            for (Map.Entry<String, KafkaFutureImpl<TopicDescription>> entry : topicFutures.entrySet()) {
                String topicName = entry.getKey();
                KafkaFutureImpl<TopicDescription> future = entry.getValue();
                Errors topicError = errors.get(topicName);
                if (topicError != null) {
                    future.completeExceptionally(topicError.exception());
                    continue;
                }
                if (!cluster.topics().contains(topicName)) {
                    future.completeExceptionally(new UnknownTopicOrPartitionException("Topic " + topicName + " not found."));
                    continue;
                }
                Uuid topicId = cluster.topicId(topicName);
                Integer authorizedOperations = response.topicAuthorizedOperations(topicName).get();
                TopicDescription topicDescription = getTopicDescriptionFromCluster(cluster, topicName, topicId, authorizedOperations);
                future.complete(topicDescription);
            }
        }

        @Override
        boolean handleUnsupportedVersionException(UnsupportedVersionException exception) {
            if (supportsDisablingTopicCreation) {
                supportsDisablingTopicCreation = false;
                return true;
            }
            return false;
        }

        @Override
        void handleFailure(Throwable throwable) {
            completeAllExceptionally(topicFutures.values(), throwable);
        }
    };
    if (!topicNamesList.isEmpty()) {
        runnable.call(call, now);
    }
    return new HashMap<>(topicFutures);
}

Example 14 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class Metadata method handleMetadataResponse.

/**
 * Transform a MetadataResponse into a new MetadataCache instance.
 */
private MetadataCache handleMetadataResponse(MetadataResponse metadataResponse, boolean isPartialUpdate, long nowMs) {
    // All encountered topics.
    Set<String> topics = new HashSet<>();
    // Retained topics to be passed to the metadata cache.
    Set<String> internalTopics = new HashSet<>();
    Set<String> unauthorizedTopics = new HashSet<>();
    Set<String> invalidTopics = new HashSet<>();
    List<MetadataResponse.PartitionMetadata> partitions = new ArrayList<>();
    Map<String, Uuid> topicIds = new HashMap<>();
    Map<String, Uuid> oldTopicIds = cache.topicIds();
    for (MetadataResponse.TopicMetadata metadata : metadataResponse.topicMetadata()) {
        String topicName = metadata.topic();
        Uuid topicId = metadata.topicId();
        topics.add(topicName);
        // We can only reason about topic ID changes when both IDs are valid, so keep oldId null unless the new metadata contains a topic ID
        Uuid oldTopicId = null;
        if (!Uuid.ZERO_UUID.equals(topicId)) {
            topicIds.put(topicName, topicId);
            oldTopicId = oldTopicIds.get(topicName);
        } else {
            topicId = null;
        }
        if (!retainTopic(topicName, metadata.isInternal(), nowMs))
            continue;
        if (metadata.isInternal())
            internalTopics.add(topicName);
        if (metadata.error() == Errors.NONE) {
            for (MetadataResponse.PartitionMetadata partitionMetadata : metadata.partitionMetadata()) {
                // Even if the partition's metadata includes an error, we need to handle
                // the update to catch new epochs
                updateLatestMetadata(partitionMetadata, metadataResponse.hasReliableLeaderEpochs(), topicId, oldTopicId).ifPresent(partitions::add);
                if (partitionMetadata.error.exception() instanceof InvalidMetadataException) {
                    log.debug("Requesting metadata update for partition {} due to error {}", partitionMetadata.topicPartition, partitionMetadata.error);
                    requestUpdate();
                }
            }
        } else {
            if (metadata.error().exception() instanceof InvalidMetadataException) {
                log.debug("Requesting metadata update for topic {} due to error {}", topicName, metadata.error());
                requestUpdate();
            }
            if (metadata.error() == Errors.INVALID_TOPIC_EXCEPTION)
                invalidTopics.add(topicName);
            else if (metadata.error() == Errors.TOPIC_AUTHORIZATION_FAILED)
                unauthorizedTopics.add(topicName);
        }
    }
    Map<Integer, Node> nodes = metadataResponse.brokersById();
    if (isPartialUpdate)
        return this.cache.mergeWith(metadataResponse.clusterId(), nodes, partitions, unauthorizedTopics, invalidTopics, internalTopics, metadataResponse.controller(), topicIds, (topic, isInternal) -> !topics.contains(topic) && retainTopic(topic, isInternal, nowMs));
    else
        return new MetadataCache(metadataResponse.clusterId(), nodes, partitions, unauthorizedTopics, invalidTopics, internalTopics, metadataResponse.controller(), topicIds);
}

Example 15 with MetadataResponse

use of org.apache.kafka.common.requests.MetadataResponse in project kafka by apache.

the class NetworkClient method handleCompletedReceives.

/**
 * Handle any completed receives and update the response list with the responses received.
 *
 * @param responses The list of responses to update
 * @param now The current time
 */
private void handleCompletedReceives(List<ClientResponse> responses, long now) {
    for (NetworkReceive receive : this.selector.completedReceives()) {
        String source = receive.source();
        InFlightRequest req = inFlightRequests.completeNext(source);
        AbstractResponse response = parseResponse(receive.payload(), req.header);
        if (throttleTimeSensor != null)
            throttleTimeSensor.record(response.throttleTimeMs(), now);
        if (log.isDebugEnabled()) {
            log.debug("Received {} response from node {} for request with header {}: {}", req.header.apiKey(), req.destination, req.header, response);
        }
        // If the received response includes a throttle delay, throttle the connection.
        maybeThrottle(response, req.header.apiVersion(), req.destination, now);
        if (req.isInternalRequest && response instanceof MetadataResponse)
            metadataUpdater.handleSuccessfulResponse(req.header, now, (MetadataResponse) response);
        else if (req.isInternalRequest && response instanceof ApiVersionsResponse)
            handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) response);
        else
            responses.add(req.completed(response, now));
    }
}