Search in sources :

Example 1 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class KafkaAdminClient method getListOffsetsCalls.

// visible for benchmark
List<Call> getListOffsetsCalls(MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> context, Map<TopicPartition, OffsetSpec> topicPartitionOffsets, Map<TopicPartition, KafkaFutureImpl<ListOffsetsResultInfo>> futures) {
    MetadataResponse mr = context.response().orElseThrow(() -> new IllegalStateException("No Metadata response"));
    Cluster clusterSnapshot = mr.buildCluster();
    List<Call> calls = new ArrayList<>();
    // grouping topic partitions per leader
    Map<Node, Map<String, ListOffsetsTopic>> leaders = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetSpec> entry : topicPartitionOffsets.entrySet()) {
        OffsetSpec offsetSpec = entry.getValue();
        TopicPartition tp = entry.getKey();
        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
        long offsetQuery = getOffsetFromOffsetSpec(offsetSpec);
        // avoid sending listOffsets request for topics with errors
        if (!mr.errors().containsKey(tp.topic())) {
            Node node = clusterSnapshot.leaderFor(tp);
            if (node != null) {
                Map<String, ListOffsetsTopic> leadersOnNode = leaders.computeIfAbsent(node, k -> new HashMap<>());
                ListOffsetsTopic topic = leadersOnNode.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
                topic.partitions().add(new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offsetQuery));
            } else {
                future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
            }
        } else {
            future.completeExceptionally(mr.errors().get(tp.topic()).exception());
        }
    }
    for (final Map.Entry<Node, Map<String, ListOffsetsTopic>> entry : leaders.entrySet()) {
        final int brokerId = entry.getKey().id();
        calls.add(new Call("listOffsets on broker " + brokerId, context.deadline(), new ConstantNodeIdProvider(brokerId)) {

            final List<ListOffsetsTopic> partitionsToQuery = new ArrayList<>(entry.getValue().values());

            private boolean supportsMaxTimestamp = partitionsToQuery.stream().flatMap(t -> t.partitions().stream()).anyMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP);

            @Override
            ListOffsetsRequest.Builder createRequest(int timeoutMs) {
                return ListOffsetsRequest.Builder.forConsumer(true, context.options().isolationLevel(), supportsMaxTimestamp).setTargetTimes(partitionsToQuery);
            }

            @Override
            void handleResponse(AbstractResponse abstractResponse) {
                ListOffsetsResponse response = (ListOffsetsResponse) abstractResponse;
                Map<TopicPartition, OffsetSpec> retryTopicPartitionOffsets = new HashMap<>();
                for (ListOffsetsTopicResponse topic : response.topics()) {
                    for (ListOffsetsPartitionResponse partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        Errors error = Errors.forCode(partition.errorCode());
                        OffsetSpec offsetRequestSpec = topicPartitionOffsets.get(tp);
                        if (offsetRequestSpec == null) {
                            log.warn("Server response mentioned unknown topic partition {}", tp);
                        } else if (MetadataOperationContext.shouldRefreshMetadata(error)) {
                            retryTopicPartitionOffsets.put(tp, offsetRequestSpec);
                        } else if (error == Errors.NONE) {
                            Optional<Integer> leaderEpoch = (partition.leaderEpoch() == ListOffsetsResponse.UNKNOWN_EPOCH) ? Optional.empty() : Optional.of(partition.leaderEpoch());
                            future.complete(new ListOffsetsResultInfo(partition.offset(), partition.timestamp(), leaderEpoch));
                        } else {
                            future.completeExceptionally(error.exception());
                        }
                    }
                }
                if (retryTopicPartitionOffsets.isEmpty()) {
                    // The server should send back a response for every topic partition. But do a sanity check anyway.
                    for (ListOffsetsTopic topic : partitionsToQuery) {
                        for (ListOffsetsPartition partition : topic.partitions()) {
                            TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                            ApiException error = new ApiException("The response from broker " + brokerId + " did not contain a result for topic partition " + tp);
                            futures.get(tp).completeExceptionally(error);
                        }
                    }
                } else {
                    Set<String> retryTopics = retryTopicPartitionOffsets.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
                    MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> retryContext = new MetadataOperationContext<>(retryTopics, context.options(), context.deadline(), futures);
                    rescheduleMetadataTask(retryContext, () -> getListOffsetsCalls(retryContext, retryTopicPartitionOffsets, futures));
                }
            }

            @Override
            void handleFailure(Throwable throwable) {
                for (ListOffsetsTopic topic : entry.getValue().values()) {
                    for (ListOffsetsPartition partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        future.completeExceptionally(throwable);
                    }
                }
            }

            @Override
            boolean handleUnsupportedVersionException(UnsupportedVersionException exception) {
                if (supportsMaxTimestamp) {
                    supportsMaxTimestamp = false;
                    // fail any unsupported futures and remove partitions from the downgraded retry
                    Iterator<ListOffsetsTopic> topicIterator = partitionsToQuery.iterator();
                    while (topicIterator.hasNext()) {
                        ListOffsetsTopic topic = topicIterator.next();
                        Iterator<ListOffsetsPartition> partitionIterator = topic.partitions().iterator();
                        while (partitionIterator.hasNext()) {
                            ListOffsetsPartition partition = partitionIterator.next();
                            if (partition.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP) {
                                futures.get(new TopicPartition(topic.name(), partition.partitionIndex())).completeExceptionally(new UnsupportedVersionException("Broker " + brokerId + " does not support MAX_TIMESTAMP offset spec"));
                                partitionIterator.remove();
                            }
                        }
                        if (topic.partitions().isEmpty()) {
                            topicIterator.remove();
                        }
                    }
                    return !partitionsToQuery.isEmpty();
                }
                return false;
            }
        });
    }
    return calls;
}
Also used : ListGroupsResponse(org.apache.kafka.common.requests.ListGroupsResponse) ThrottlingQuotaExceededException(org.apache.kafka.common.errors.ThrottlingQuotaExceededException) ListOffsetsRequest(org.apache.kafka.common.requests.ListOffsetsRequest) UserName(org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData.UserName) ListPartitionReassignmentsTopics(org.apache.kafka.common.message.ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics) ClientQuotaFilter(org.apache.kafka.common.quota.ClientQuotaFilter) KafkaException(org.apache.kafka.common.KafkaException) AppInfoParser(org.apache.kafka.common.utils.AppInfoParser) ClientUtils(org.apache.kafka.clients.ClientUtils) DescribeClientQuotasRequest(org.apache.kafka.common.requests.DescribeClientQuotasRequest) ListGroupsRequestData(org.apache.kafka.common.message.ListGroupsRequestData) AclCreationResult(org.apache.kafka.common.message.CreateAclsResponseData.AclCreationResult) Duration(java.time.Duration) Map(java.util.Map) TopicNameCollection(org.apache.kafka.common.TopicCollection.TopicNameCollection) UpdateFeaturesRequestData(org.apache.kafka.common.message.UpdateFeaturesRequestData) Utils.closeQuietly(org.apache.kafka.common.utils.Utils.closeQuietly) DeleteAclsFilter(org.apache.kafka.common.message.DeleteAclsRequestData.DeleteAclsFilter) CreateAclsRequest(org.apache.kafka.common.requests.CreateAclsRequest) DescribeConfigsRequestData(org.apache.kafka.common.message.DescribeConfigsRequestData) DescribableLogDirTopic(org.apache.kafka.common.message.DescribeLogDirsRequestData.DescribableLogDirTopic) CommonClientConfigs(org.apache.kafka.clients.CommonClientConfigs) Sensor(org.apache.kafka.common.metrics.Sensor) DescribeLogDirsResponseData(org.apache.kafka.common.message.DescribeLogDirsResponseData) ClientQuotaAlteration(org.apache.kafka.common.quota.ClientQuotaAlteration) AlterReplicaLogDirTopicResult(org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirTopicResult) DeleteTopicsRequestData(org.apache.kafka.common.message.DeleteTopicsRequestData) AlterReplicaLogDirsRequest(org.apache.kafka.common.requests.AlterReplicaLogDirsRequest) ListGroupsRequest(org.apache.kafka.common.requests.ListGroupsRequest) Metrics(org.apache.kafka.common.metrics.Metrics) Stream(java.util.stream.Stream) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Errors(org.apache.kafka.common.protocol.Errors) Node(org.apache.kafka.common.Node) UnknownTopicOrPartitionException(org.apache.kafka.common.errors.UnknownTopicOrPartitionException) UnacceptableCredentialException(org.apache.kafka.common.errors.UnacceptableCredentialException) AuthenticationException(org.apache.kafka.common.errors.AuthenticationException) DeleteTopicState(org.apache.kafka.common.message.DeleteTopicsRequestData.DeleteTopicState) SupportedFeatureKey(org.apache.kafka.common.message.ApiVersionsResponseData.SupportedFeatureKey) DeleteRecordsTopic(org.apache.kafka.common.message.DeleteRecordsRequestData.DeleteRecordsTopic) AbstractResponse(org.apache.kafka.common.requests.AbstractResponse) AlterUserScramCredentialsResponse(org.apache.kafka.common.requests.AlterUserScramCredentialsResponse) DescribeClusterResponse(org.apache.kafka.common.requests.DescribeClusterResponse) DelegationToken(org.apache.kafka.common.security.token.delegation.DelegationToken) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) Supplier(java.util.function.Supplier) DescribeAclsResponse(org.apache.kafka.common.requests.DescribeAclsResponse) AlterReplicaLogDirsResponse(org.apache.kafka.common.requests.AlterReplicaLogDirsResponse) AlterConsumerGroupOffsetsHandler(org.apache.kafka.clients.admin.internals.AlterConsumerGroupOffsetsHandler) UpdateFeaturesRequest(org.apache.kafka.common.requests.UpdateFeaturesRequest) KafkaStorageException(org.apache.kafka.common.errors.KafkaStorageException) AlterPartitionReassignmentsRequest(org.apache.kafka.common.requests.AlterPartitionReassignmentsRequest) DeleteAclsMatchingAcl(org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl) DescribeLogDirsRequestData(org.apache.kafka.common.message.DescribeLogDirsRequestData) ApiVersions(org.apache.kafka.clients.ApiVersions) TopicMetadataAndConfig(org.apache.kafka.clients.admin.CreateTopicsResult.TopicMetadataAndConfig) IncrementalAlterConfigsRequest(org.apache.kafka.common.requests.IncrementalAlterConfigsRequest) AtomicLong(java.util.concurrent.atomic.AtomicLong) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) ReassignableTopic(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignableTopic) TreeMap(java.util.TreeMap) IncrementalAlterConfigsResponse(org.apache.kafka.common.requests.IncrementalAlterConfigsResponse) UpdatableFeatureResult(org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResult) CreatePartitionsAssignment(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment) ElectLeadersRequest(org.apache.kafka.common.requests.ElectLeadersRequest) FilterResults(org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults) DescribeUserScramCredentialsResponse(org.apache.kafka.common.requests.DescribeUserScramCredentialsResponse) AdminApiFuture(org.apache.kafka.clients.admin.internals.AdminApiFuture) AbstractRequest(org.apache.kafka.common.requests.AbstractRequest) ChannelBuilder(org.apache.kafka.common.network.ChannelBuilder) MemberIdentity(org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity) TokenInformation(org.apache.kafka.common.security.token.delegation.TokenInformation) CreateDelegationTokenRequestData(org.apache.kafka.common.message.CreateDelegationTokenRequestData) CreateAclsRequestData(org.apache.kafka.common.message.CreateAclsRequestData) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) KafkaClient(org.apache.kafka.clients.KafkaClient) DescribeLogDirsRequest(org.apache.kafka.common.requests.DescribeLogDirsRequest) DescribeConfigsResponseData(org.apache.kafka.common.message.DescribeConfigsResponseData) TopicIdCollection(org.apache.kafka.common.TopicCollection.TopicIdCollection) ListOffsetsTopic(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic) TopicPartition(org.apache.kafka.common.TopicPartition) Time(org.apache.kafka.common.utils.Time) Collection(java.util.Collection) AlterReplicaLogDir(org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDir) CreateAclsResponse(org.apache.kafka.common.requests.CreateAclsResponse) DescribeClusterRequest(org.apache.kafka.common.requests.DescribeClusterRequest) DescribeUserScramCredentialsResponseData(org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData) Collectors(java.util.stream.Collectors) DescribeConfigsRequest(org.apache.kafka.common.requests.DescribeConfigsRequest) Objects(java.util.Objects) CreatePartitionsResponse(org.apache.kafka.common.requests.CreatePartitionsResponse) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) UnregisterBrokerRequestData(org.apache.kafka.common.message.UnregisterBrokerRequestData) AlterUserScramCredentialsRequest(org.apache.kafka.common.requests.AlterUserScramCredentialsRequest) RenewDelegationTokenRequestData(org.apache.kafka.common.message.RenewDelegationTokenRequestData) ExpireDelegationTokenRequest(org.apache.kafka.common.requests.ExpireDelegationTokenRequest) Uuid(org.apache.kafka.common.Uuid) DeleteTopicsRequest(org.apache.kafka.common.requests.DeleteTopicsRequest) ReassignablePartitionResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse) DeleteRecordsRequestData(org.apache.kafka.common.message.DeleteRecordsRequestData) FinalizedFeatureKey(org.apache.kafka.common.message.ApiVersionsResponseData.FinalizedFeatureKey) DeleteAclsResponse(org.apache.kafka.common.requests.DeleteAclsResponse) DeleteAclsFilterResult(org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult) AlterPartitionReassignmentsResponse(org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse) DescribeConfigsResponse(org.apache.kafka.common.requests.DescribeConfigsResponse) Function(java.util.function.Function) UnknownServerException(org.apache.kafka.common.errors.UnknownServerException) DeleteRecordsResponseData(org.apache.kafka.common.message.DeleteRecordsResponseData) DescribeUserScramCredentialsRequest(org.apache.kafka.common.requests.DescribeUserScramCredentialsRequest) HashSet(java.util.HashSet) CreatableTopicConfigs(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicConfigs) TopicCollection(org.apache.kafka.common.TopicCollection) OngoingPartitionReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment) LinkedList(java.util.LinkedList) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Logger(org.slf4j.Logger) AllBrokersStrategy(org.apache.kafka.clients.admin.internals.AllBrokersStrategy) AlterConfigsResponse(org.apache.kafka.common.requests.AlterConfigsResponse) CreateTopicsResponse(org.apache.kafka.common.requests.CreateTopicsResponse) ConfigException(org.apache.kafka.common.config.ConfigException) DeleteRecordsPartition(org.apache.kafka.common.message.DeleteRecordsRequestData.DeleteRecordsPartition) KafkaThread(org.apache.kafka.common.utils.KafkaThread) AlterConfigsRequest(org.apache.kafka.common.requests.AlterConfigsRequest) RenewDelegationTokenRequest(org.apache.kafka.common.requests.RenewDelegationTokenRequest) DefaultHostResolver(org.apache.kafka.clients.DefaultHostResolver) DisconnectException(org.apache.kafka.common.errors.DisconnectException) InvalidRequestException(org.apache.kafka.common.errors.InvalidRequestException) KafkaPrincipal(org.apache.kafka.common.security.auth.KafkaPrincipal) Comparator(java.util.Comparator) ExpireDelegationTokenResponse(org.apache.kafka.common.requests.ExpireDelegationTokenResponse) ReplicaLogDirInfo(org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo) Arrays(java.util.Arrays) ApiVersionsRequest(org.apache.kafka.common.requests.ApiVersionsRequest) DescribeClusterRequestData(org.apache.kafka.common.message.DescribeClusterRequestData) CreatePartitionsTopicCollection(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopicCollection) DeleteRecordsTopicResult(org.apache.kafka.common.message.DeleteRecordsResponseData.DeleteRecordsTopicResult) Cluster(org.apache.kafka.common.Cluster) AbortTransactionHandler(org.apache.kafka.clients.admin.internals.AbortTransactionHandler) AdminApiHandler(org.apache.kafka.clients.admin.internals.AdminApiHandler) UnregisterBrokerRequest(org.apache.kafka.common.requests.UnregisterBrokerRequest) CreatePartitionsTopic(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic) RemoveMembersFromConsumerGroupHandler(org.apache.kafka.clients.admin.internals.RemoveMembersFromConsumerGroupHandler) ApiVersionsResponse(org.apache.kafka.common.requests.ApiVersionsResponse) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) LogContext(org.apache.kafka.common.utils.LogContext) ListPartitionReassignmentsRequestData(org.apache.kafka.common.message.ListPartitionReassignmentsRequestData) CreatePartitionsRequestData(org.apache.kafka.common.message.CreatePartitionsRequestData) TimestampSpec(org.apache.kafka.clients.admin.OffsetSpec.TimestampSpec) ListTransactionsHandler(org.apache.kafka.clients.admin.internals.ListTransactionsHandler) DeleteAclsRequest(org.apache.kafka.common.requests.DeleteAclsRequest) InvalidTopicException(org.apache.kafka.common.errors.InvalidTopicException) Set(java.util.Set) PartitionInfo(org.apache.kafka.common.PartitionInfo) DeleteRecordsRequest(org.apache.kafka.common.requests.DeleteRecordsRequest) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) CoordinatorKey(org.apache.kafka.clients.admin.internals.CoordinatorKey) MetricsReporter(org.apache.kafka.common.metrics.MetricsReporter) ConsumerGroupState(org.apache.kafka.common.ConsumerGroupState) InvalidKeyException(java.security.InvalidKeyException) CreatableTopicResult(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult) MetadataResponse(org.apache.kafka.common.requests.MetadataResponse) AlterUserScramCredentialsRequestData(org.apache.kafka.common.message.AlterUserScramCredentialsRequestData) CreatableRenewers(org.apache.kafka.common.message.CreateDelegationTokenRequestData.CreatableRenewers) Selector(org.apache.kafka.common.network.Selector) KafkaMetricsContext(org.apache.kafka.common.metrics.KafkaMetricsContext) RenewDelegationTokenResponse(org.apache.kafka.common.requests.RenewDelegationTokenResponse) RetriableException(org.apache.kafka.common.errors.RetriableException) ExpireDelegationTokenRequestData(org.apache.kafka.common.message.ExpireDelegationTokenRequestData) ArrayList(java.util.ArrayList) MetricsContext(org.apache.kafka.common.metrics.MetricsContext) ElectionType(org.apache.kafka.common.ElectionType) NetworkClient(org.apache.kafka.clients.NetworkClient) InterfaceStability(org.apache.kafka.common.annotation.InterfaceStability) AlterPartitionReassignmentsRequestData(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) MetadataOperationContext(org.apache.kafka.clients.admin.internals.MetadataOperationContext) ListGroupsResponseData(org.apache.kafka.common.message.ListGroupsResponseData) TopicPartitionInfo(org.apache.kafka.common.TopicPartitionInfo) ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramFormatter) AlterClientQuotasResponse(org.apache.kafka.common.requests.AlterClientQuotasResponse) AclCreation(org.apache.kafka.common.message.CreateAclsRequestData.AclCreation) TopicPartitionReplica(org.apache.kafka.common.TopicPartitionReplica) DeleteRecordsResponse(org.apache.kafka.common.requests.DeleteRecordsResponse) ReassignablePartition(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition) AclOperation(org.apache.kafka.common.acl.AclOperation) AlterReplicaLogDirsRequestData(org.apache.kafka.common.message.AlterReplicaLogDirsRequestData) UnsupportedSaslMechanismException(org.apache.kafka.common.errors.UnsupportedSaslMechanismException) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) DescribeUserScramCredentialsRequestData(org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException) MetadataRequestData(org.apache.kafka.common.message.MetadataRequestData) DescribeDelegationTokenResponse(org.apache.kafka.common.requests.DescribeDelegationTokenResponse) AdminApiDriver(org.apache.kafka.clients.admin.internals.AdminApiDriver) AlterClientQuotasRequest(org.apache.kafka.common.requests.AlterClientQuotasRequest) CreatePartitionsTopicResult(org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult) SimpleAdminApiFuture(org.apache.kafka.clients.admin.internals.AdminApiFuture.SimpleAdminApiFuture) AlterReplicaLogDirPartitionResult(org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirPartitionResult) ListPartitionReassignmentsResponse(org.apache.kafka.common.requests.ListPartitionReassignmentsResponse) ClientRequest(org.apache.kafka.clients.ClientRequest) DescribeClientQuotasResponse(org.apache.kafka.common.requests.DescribeClientQuotasResponse) UpdateFeaturesResponse(org.apache.kafka.common.requests.UpdateFeaturesResponse) ReassignableTopicResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse) DeleteAclsRequestData(org.apache.kafka.common.message.DeleteAclsRequestData) UnregisterBrokerResponse(org.apache.kafka.common.requests.UnregisterBrokerResponse) CreatePartitionsRequest(org.apache.kafka.common.requests.CreatePartitionsRequest) MetadataRequest.convertToMetadataRequestTopic(org.apache.kafka.common.requests.MetadataRequest.convertToMetadataRequestTopic) Metric(org.apache.kafka.common.Metric) MetricName(org.apache.kafka.common.MetricName) AdminMetadataManager(org.apache.kafka.clients.admin.internals.AdminMetadataManager) DescribeAclsRequest(org.apache.kafka.common.requests.DescribeAclsRequest) Predicate(java.util.function.Predicate) MetricConfig(org.apache.kafka.common.metrics.MetricConfig) KafkaFuture(org.apache.kafka.common.KafkaFuture) InetSocketAddress(java.net.InetSocketAddress) ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) List(java.util.List) DeletableTopicResult(org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult) Optional(java.util.Optional) ClientResponse(org.apache.kafka.clients.ClientResponse) ConsumerProtocol(org.apache.kafka.clients.consumer.internals.ConsumerProtocol) ListConsumerGroupOffsetsHandler(org.apache.kafka.clients.admin.internals.ListConsumerGroupOffsetsHandler) MetadataRequest.convertTopicIdsToMetadataRequestTopic(org.apache.kafka.common.requests.MetadataRequest.convertTopicIdsToMetadataRequestTopic) CreateTopicsRequest(org.apache.kafka.common.requests.CreateTopicsRequest) ElectLeadersResponse(org.apache.kafka.common.requests.ElectLeadersResponse) StaleMetadataException(org.apache.kafka.clients.StaleMetadataException) AclBindingFilter(org.apache.kafka.common.acl.AclBindingFilter) HashMap(java.util.HashMap) DescribeConsumerGroupsHandler(org.apache.kafka.clients.admin.internals.DescribeConsumerGroupsHandler) ApiError(org.apache.kafka.common.requests.ApiError) ConfigResource(org.apache.kafka.common.config.ConfigResource) CreateDelegationTokenResponse(org.apache.kafka.common.requests.CreateDelegationTokenResponse) CreateDelegationTokenResponseData(org.apache.kafka.common.message.CreateDelegationTokenResponseData) DeleteConsumerGroupsHandler(org.apache.kafka.clients.admin.internals.DeleteConsumerGroupsHandler) MetadataRequest(org.apache.kafka.common.requests.MetadataRequest) AclBinding(org.apache.kafka.common.acl.AclBinding) ClientQuotaEntity(org.apache.kafka.common.quota.ClientQuotaEntity) DescribeProducersHandler(org.apache.kafka.clients.admin.internals.DescribeProducersHandler) OngoingTopicReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment) ListPartitionReassignmentsRequest(org.apache.kafka.common.requests.ListPartitionReassignmentsRequest) DeleteConsumerGroupOffsetsHandler(org.apache.kafka.clients.admin.internals.DeleteConsumerGroupOffsetsHandler) AlterReplicaLogDirTopic(org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDirTopic) JmxReporter(org.apache.kafka.common.metrics.JmxReporter) Utils(org.apache.kafka.common.utils.Utils) DescribeTransactionsHandler(org.apache.kafka.clients.admin.internals.DescribeTransactionsHandler) ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) Iterator(java.util.Iterator) FilterResult(org.apache.kafka.clients.admin.DeleteAclsResult.FilterResult) DeleteAclsResponseData(org.apache.kafka.common.message.DeleteAclsResponseData) TimeUnit(java.util.concurrent.TimeUnit) DescribeDelegationTokenRequest(org.apache.kafka.common.requests.DescribeDelegationTokenRequest) HostResolver(org.apache.kafka.clients.HostResolver) DeleteTopicsResponse(org.apache.kafka.common.requests.DeleteTopicsResponse) DescribeLogDirsResponse(org.apache.kafka.common.requests.DescribeLogDirsResponse) Collections(java.util.Collections) ApiException(org.apache.kafka.common.errors.ApiException) CreateDelegationTokenRequest(org.apache.kafka.common.requests.CreateDelegationTokenRequest) HashSet(java.util.HashSet) Set(java.util.Set) ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) HashMap(java.util.HashMap) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Node(org.apache.kafka.common.Node) ChannelBuilder(org.apache.kafka.common.network.ChannelBuilder) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) ArrayList(java.util.ArrayList) ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) ListOffsetsTopic(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic) MetadataResponse(org.apache.kafka.common.requests.MetadataResponse) Iterator(java.util.Iterator) Optional(java.util.Optional) AbstractResponse(org.apache.kafka.common.requests.AbstractResponse) Cluster(org.apache.kafka.common.Cluster) MetadataOperationContext(org.apache.kafka.clients.admin.internals.MetadataOperationContext) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) Errors(org.apache.kafka.common.protocol.Errors) TopicPartition(org.apache.kafka.common.TopicPartition) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) Map(java.util.Map) TreeMap(java.util.TreeMap) HashMap(java.util.HashMap) ApiException(org.apache.kafka.common.errors.ApiException) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException)

Example 2 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class KafkaConsumerTest method listOffsetsResponse.

private ListOffsetsResponse listOffsetsResponse(Map<TopicPartition, Long> partitionOffsets, Map<TopicPartition, Errors> partitionErrors) {
    Map<String, ListOffsetsTopicResponse> responses = new HashMap<>();
    for (Map.Entry<TopicPartition, Long> partitionOffset : partitionOffsets.entrySet()) {
        TopicPartition tp = partitionOffset.getKey();
        ListOffsetsTopicResponse topic = responses.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopicResponse().setName(tp.topic()));
        topic.partitions().add(new ListOffsetsPartitionResponse().setPartitionIndex(tp.partition()).setErrorCode(Errors.NONE.code()).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP).setOffset(partitionOffset.getValue()));
    }
    for (Map.Entry<TopicPartition, Errors> partitionError : partitionErrors.entrySet()) {
        TopicPartition tp = partitionError.getKey();
        ListOffsetsTopicResponse topic = responses.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopicResponse().setName(tp.topic()));
        topic.partitions().add(new ListOffsetsPartitionResponse().setPartitionIndex(tp.partition()).setErrorCode(partitionError.getValue().code()).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP).setOffset(ListOffsetsResponse.UNKNOWN_OFFSET));
    }
    ListOffsetsResponseData data = new ListOffsetsResponseData().setTopics(new ArrayList<>(responses.values()));
    return new ListOffsetsResponse(data);
}
Also used : LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) ListOffsetsResponseData(org.apache.kafka.common.message.ListOffsetsResponseData) Errors(org.apache.kafka.common.protocol.Errors) TopicPartition(org.apache.kafka.common.TopicPartition) OptionalLong(java.util.OptionalLong) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) Collections.singletonMap(java.util.Collections.singletonMap) AbstractMap(java.util.AbstractMap)

Example 3 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class SaslAuthenticatorTest method testConvertListOffsetResponseToSaslHandshakeResponse.

@Test
public void testConvertListOffsetResponseToSaslHandshakeResponse() {
    ListOffsetsResponseData data = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Collections.singletonList(new ListOffsetsTopicResponse().setName("topic").setPartitions(Collections.singletonList(new ListOffsetsPartitionResponse().setErrorCode(Errors.NONE.code()).setLeaderEpoch(ListOffsetsResponse.UNKNOWN_EPOCH).setPartitionIndex(0).setOffset(0).setTimestamp(0)))));
    ListOffsetsResponse response = new ListOffsetsResponse(data);
    ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(response, LIST_OFFSETS.latestVersion(), 0);
    final RequestHeader header0 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID);
    assertThrows(SchemaException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header0));
    final RequestHeader header1 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", 1);
    assertThrows(IllegalStateException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header1));
}
Also used : ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) RequestHeader(org.apache.kafka.common.requests.RequestHeader) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) ListOffsetsResponseData(org.apache.kafka.common.message.ListOffsetsResponseData) ByteBuffer(java.nio.ByteBuffer) Test(org.junit.jupiter.api.Test)

Example 4 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class KafkaAdminClientTest method testListOffsetsMaxTimestampUnsupportedMultipleOffsetSpec.

@Test
public void testListOffsetsMaxTimestampUnsupportedMultipleOffsetSpec() throws Exception {
    Node node = new Node(0, "localhost", 8120);
    List<Node> nodes = Collections.singletonList(node);
    List<PartitionInfo> pInfos = new ArrayList<>();
    pInfos.add(new PartitionInfo("foo", 0, node, new Node[] { node }, new Node[] { node }));
    pInfos.add(new PartitionInfo("foo", 1, node, new Node[] { node }, new Node[] { node }));
    final Cluster cluster = new Cluster("mockClusterId", nodes, pInfos, Collections.emptySet(), Collections.emptySet(), node);
    final TopicPartition tp0 = new TopicPartition("foo", 0);
    final TopicPartition tp1 = new TopicPartition("foo", 1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster, AdminClientConfig.RETRIES_CONFIG, "2")) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create(ApiKeys.LIST_OFFSETS.id, (short) 0, (short) 6));
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE));
        // listoffsets response from broker 0
        env.kafkaClient().prepareUnsupportedVersionResponse(request -> request instanceof ListOffsetsRequest);
        ListOffsetsTopicResponse topicResponse = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp1, Errors.NONE, -1L, 345L, 543);
        ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(topicResponse));
        env.kafkaClient().prepareResponseFrom(// ensure that no max timestamp requests are retried
        request -> request instanceof ListOffsetsRequest && ((ListOffsetsRequest) request).topics().stream().flatMap(t -> t.partitions().stream()).noneMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP), new ListOffsetsResponse(responseData), node);
        ListOffsetsResult result = env.adminClient().listOffsets(new HashMap<TopicPartition, OffsetSpec>() {

            {
                put(tp0, OffsetSpec.maxTimestamp());
                put(tp1, OffsetSpec.latest());
            }
        });
        TestUtils.assertFutureThrows(result.partitionResult(tp0), UnsupportedVersionException.class);
        ListOffsetsResultInfo tp1Offset = result.partitionResult(tp1).get();
        assertEquals(345L, tp1Offset.offset());
        assertEquals(543, tp1Offset.leaderEpoch().get().intValue());
        assertEquals(-1L, tp1Offset.timestamp());
    }
}
Also used : ListGroupsResponse(org.apache.kafka.common.requests.ListGroupsResponse) ThrottlingQuotaExceededException(org.apache.kafka.common.errors.ThrottlingQuotaExceededException) ListOffsetsRequest(org.apache.kafka.common.requests.ListOffsetsRequest) ClientQuotaFilter(org.apache.kafka.common.quota.ClientQuotaFilter) AclPermissionType(org.apache.kafka.common.acl.AclPermissionType) LeaveGroupRequest(org.apache.kafka.common.requests.LeaveGroupRequest) KafkaException(org.apache.kafka.common.KafkaException) ListTransactionsRequest(org.apache.kafka.common.requests.ListTransactionsRequest) Collections.singletonList(java.util.Collections.singletonList) ClientUtils(org.apache.kafka.clients.ClientUtils) Future(java.util.concurrent.Future) DescribeTransactionsRequest(org.apache.kafka.common.requests.DescribeTransactionsRequest) PartitionResult(org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) DeletableTopicResultCollection(org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResultCollection) DescribeLogDirsResponseData(org.apache.kafka.common.message.DescribeLogDirsResponseData) ClientQuotaAlteration(org.apache.kafka.common.quota.ClientQuotaAlteration) AlterReplicaLogDirTopicResult(org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirTopicResult) AlterReplicaLogDirsResponseData(org.apache.kafka.common.message.AlterReplicaLogDirsResponseData) ListGroupsRequest(org.apache.kafka.common.requests.ListGroupsRequest) CountDownLatch(java.util.concurrent.CountDownLatch) DeleteTopicsResponseData(org.apache.kafka.common.message.DeleteTopicsResponseData) Stream(java.util.stream.Stream) ApiMessageType(org.apache.kafka.common.message.ApiMessageType) MetadataResponsePartition(org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition) DescribeGroupsResponse(org.apache.kafka.common.requests.DescribeGroupsResponse) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Errors(org.apache.kafka.common.protocol.Errors) Node(org.apache.kafka.common.Node) UnknownTopicOrPartitionException(org.apache.kafka.common.errors.UnknownTopicOrPartitionException) DescribedGroupMember(org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember) NodeApiVersions(org.apache.kafka.clients.NodeApiVersions) AuthenticationException(org.apache.kafka.common.errors.AuthenticationException) Assertions.fail(org.junit.jupiter.api.Assertions.fail) AlterUserScramCredentialsResponseData(org.apache.kafka.common.message.AlterUserScramCredentialsResponseData) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) CreatableTopicResultCollection(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResultCollection) AlterUserScramCredentialsResponse(org.apache.kafka.common.requests.AlterUserScramCredentialsResponse) ClientDnsLookup(org.apache.kafka.clients.ClientDnsLookup) DescribeClusterResponse(org.apache.kafka.common.requests.DescribeClusterResponse) WriteTxnMarkersRequest(org.apache.kafka.common.requests.WriteTxnMarkersRequest) DescribeClusterBroker(org.apache.kafka.common.message.DescribeClusterResponseData.DescribeClusterBroker) DescribeAclsResponse(org.apache.kafka.common.requests.DescribeAclsResponse) WriteTxnMarkersResponseData(org.apache.kafka.common.message.WriteTxnMarkersResponseData) SecurityDisabledException(org.apache.kafka.common.errors.SecurityDisabledException) AlterReplicaLogDirsResponse(org.apache.kafka.common.requests.AlterReplicaLogDirsResponse) OptionalLong(java.util.OptionalLong) UnregisterBrokerResponseData(org.apache.kafka.common.message.UnregisterBrokerResponseData) UpdateFeaturesRequest(org.apache.kafka.common.requests.UpdateFeaturesRequest) OffsetCommitResponse(org.apache.kafka.common.requests.OffsetCommitResponse) DescribeTransactionsResponseData(org.apache.kafka.common.message.DescribeTransactionsResponseData) ResourcePattern(org.apache.kafka.common.resource.ResourcePattern) OffsetDeleteResponsePartitionCollection(org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) IncrementalAlterConfigsResponse(org.apache.kafka.common.requests.IncrementalAlterConfigsResponse) MemberResponse(org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse) FilterResults(org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults) JoinGroupRequest(org.apache.kafka.common.requests.JoinGroupRequest) GroupAuthorizationException(org.apache.kafka.common.errors.GroupAuthorizationException) DescribeUserScramCredentialsResponse(org.apache.kafka.common.requests.DescribeUserScramCredentialsResponse) CredentialInfo(org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData.CredentialInfo) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) ClusterAuthorizationException(org.apache.kafka.common.errors.ClusterAuthorizationException) MemberIdentity(org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity) DeletableGroupResult(org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult) LeaveGroupRequestData(org.apache.kafka.common.message.LeaveGroupRequestData) ReplicaElectionResult(org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult) DescribeConfigsResponseData(org.apache.kafka.common.message.DescribeConfigsResponseData) ApiVersionsResponseData(org.apache.kafka.common.message.ApiVersionsResponseData) TopicPartition(org.apache.kafka.common.TopicPartition) Time(org.apache.kafka.common.utils.Time) Collection(java.util.Collection) CreateAclsResponse(org.apache.kafka.common.requests.CreateAclsResponse) DescribeClusterRequest(org.apache.kafka.common.requests.DescribeClusterRequest) DescribeUserScramCredentialsResponseData(org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData) DescribeTransactionsResponse(org.apache.kafka.common.requests.DescribeTransactionsResponse) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) ResourceType(org.apache.kafka.common.resource.ResourceType) CreatePartitionsResponse(org.apache.kafka.common.requests.CreatePartitionsResponse) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) Uuid(org.apache.kafka.common.Uuid) DeleteTopicsRequest(org.apache.kafka.common.requests.DeleteTopicsRequest) ReassignablePartitionResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse) OffsetDeleteResponseTopicCollection(org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection) DeleteAclsResponse(org.apache.kafka.common.requests.DeleteAclsResponse) ListPartitionReassignmentsResponseData(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData) AlterPartitionReassignmentsResponse(org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse) CompletableFuture(java.util.concurrent.CompletableFuture) DescribeConfigsResponse(org.apache.kafka.common.requests.DescribeConfigsResponse) PatternType(org.apache.kafka.common.resource.PatternType) OptionalInt(java.util.OptionalInt) LeaveGroupResponse(org.apache.kafka.common.requests.LeaveGroupResponse) UnknownServerException(org.apache.kafka.common.errors.UnknownServerException) DeleteRecordsResponseData(org.apache.kafka.common.message.DeleteRecordsResponseData) HashSet(java.util.HashSet) AccessControlEntry(org.apache.kafka.common.acl.AccessControlEntry) AlterConfigsResourceResponse(org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse) TopicCollection(org.apache.kafka.common.TopicCollection) OngoingPartitionReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment) UnknownMemberIdException(org.apache.kafka.common.errors.UnknownMemberIdException) TimeoutException(org.apache.kafka.common.errors.TimeoutException) DeletableGroupResultCollection(org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResultCollection) MockClient(org.apache.kafka.clients.MockClient) Logger(org.slf4j.Logger) GroupSubscribedToTopicException(org.apache.kafka.common.errors.GroupSubscribedToTopicException) CreateTopicsResponse(org.apache.kafka.common.requests.CreateTopicsResponse) Semaphore(java.util.concurrent.Semaphore) LeaveGroupResponseData(org.apache.kafka.common.message.LeaveGroupResponseData) ConfigException(org.apache.kafka.common.config.ConfigException) TopicAuthorizationException(org.apache.kafka.common.errors.TopicAuthorizationException) InvalidRequestException(org.apache.kafka.common.errors.InvalidRequestException) MockTime(org.apache.kafka.common.utils.MockTime) Arrays(java.util.Arrays) ApiVersionsRequest(org.apache.kafka.common.requests.ApiVersionsRequest) NotLeaderOrFollowerException(org.apache.kafka.common.errors.NotLeaderOrFollowerException) Cluster(org.apache.kafka.common.Cluster) ApiVersionsResponse(org.apache.kafka.common.requests.ApiVersionsResponse) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) DescribeAclsResponseData(org.apache.kafka.common.message.DescribeAclsResponseData) TestUtils(org.apache.kafka.test.TestUtils) InvalidTopicException(org.apache.kafka.common.errors.InvalidTopicException) Set(java.util.Set) PartitionInfo(org.apache.kafka.common.PartitionInfo) DeleteGroupsResponse(org.apache.kafka.common.requests.DeleteGroupsResponse) ListTransactionsResponse(org.apache.kafka.common.requests.ListTransactionsResponse) TopicDeletionDisabledException(org.apache.kafka.common.errors.TopicDeletionDisabledException) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) ConsumerGroupState(org.apache.kafka.common.ConsumerGroupState) CreatableTopicResult(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult) MetadataResponse(org.apache.kafka.common.requests.MetadataResponse) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) DescribeGroupsResponseData(org.apache.kafka.common.message.DescribeGroupsResponseData) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) RequestTestUtils(org.apache.kafka.common.requests.RequestTestUtils) ArrayList(java.util.ArrayList) Features(org.apache.kafka.common.feature.Features) OffsetDeleteResponsePartition(org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartition) CreateAclsResponseData(org.apache.kafka.common.message.CreateAclsResponseData) ElectionType(org.apache.kafka.common.ElectionType) FindCoordinatorResponseData(org.apache.kafka.common.message.FindCoordinatorResponseData) MetadataResponseTopic(org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic) ListTransactionsResponseData(org.apache.kafka.common.message.ListTransactionsResponseData) FindCoordinatorRequest(org.apache.kafka.common.requests.FindCoordinatorRequest) OffsetFetchResponse(org.apache.kafka.common.requests.OffsetFetchResponse) UnknownTopicIdException(org.apache.kafka.common.errors.UnknownTopicIdException) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) ListGroupsResponseData(org.apache.kafka.common.message.ListGroupsResponseData) SaslAuthenticationException(org.apache.kafka.common.errors.SaslAuthenticationException) ValueSource(org.junit.jupiter.params.provider.ValueSource) AlterClientQuotasResponse(org.apache.kafka.common.requests.AlterClientQuotasResponse) TopicPartitionReplica(org.apache.kafka.common.TopicPartitionReplica) FencedInstanceIdException(org.apache.kafka.common.errors.FencedInstanceIdException) DeleteRecordsResponse(org.apache.kafka.common.requests.DeleteRecordsResponse) LEAVE_GROUP_REASON(org.apache.kafka.clients.admin.KafkaAdminClient.LEAVE_GROUP_REASON) AclOperation(org.apache.kafka.common.acl.AclOperation) DescribeClusterResponseData(org.apache.kafka.common.message.DescribeClusterResponseData) DeleteGroupsResponseData(org.apache.kafka.common.message.DeleteGroupsResponseData) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException) WriteTxnMarkersResponse(org.apache.kafka.common.requests.WriteTxnMarkersResponse) ResourcePatternFilter(org.apache.kafka.common.resource.ResourcePatternFilter) CreatePartitionsResponseData(org.apache.kafka.common.message.CreatePartitionsResponseData) FindCoordinatorResponse(org.apache.kafka.common.requests.FindCoordinatorResponse) LoggerFactory(org.slf4j.LoggerFactory) CreatePartitionsTopicResult(org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult) AlterReplicaLogDirPartitionResult(org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirPartitionResult) ListPartitionReassignmentsResponse(org.apache.kafka.common.requests.ListPartitionReassignmentsResponse) ByteBuffer(java.nio.ByteBuffer) DescribeClientQuotasResponse(org.apache.kafka.common.requests.DescribeClientQuotasResponse) UpdateFeaturesResponse(org.apache.kafka.common.requests.UpdateFeaturesResponse) ReassignableTopicResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse) DescribeProducersResponse(org.apache.kafka.common.requests.DescribeProducersResponse) Collections.singleton(java.util.Collections.singleton) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) UnregisterBrokerResponse(org.apache.kafka.common.requests.UnregisterBrokerResponse) CreatePartitionsRequest(org.apache.kafka.common.requests.CreatePartitionsRequest) RecordVersion(org.apache.kafka.common.record.RecordVersion) Collections.emptyList(java.util.Collections.emptyList) KafkaFuture(org.apache.kafka.common.KafkaFuture) LogDirNotFoundException(org.apache.kafka.common.errors.LogDirNotFoundException) InetSocketAddress(java.net.InetSocketAddress) ListOffsetsResponseData(org.apache.kafka.common.message.ListOffsetsResponseData) ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) TopicExistsException(org.apache.kafka.common.errors.TopicExistsException) List(java.util.List) DescribeProducersResponseData(org.apache.kafka.common.message.DescribeProducersResponseData) DeletableTopicResult(org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult) OffsetDeleteResponseData(org.apache.kafka.common.message.OffsetDeleteResponseData) Optional(java.util.Optional) CreateTopicsResponseData(org.apache.kafka.common.message.CreateTopicsResponseData) ConsumerProtocol(org.apache.kafka.clients.consumer.internals.ConsumerProtocol) ClientQuotaFilterComponent(org.apache.kafka.common.quota.ClientQuotaFilterComponent) CreateTopicsRequest(org.apache.kafka.common.requests.CreateTopicsRequest) ElectLeadersResponse(org.apache.kafka.common.requests.ElectLeadersResponse) AclBindingFilter(org.apache.kafka.common.acl.AclBindingFilter) DescribeLogDirsTopic(org.apache.kafka.common.message.DescribeLogDirsResponseData.DescribeLogDirsTopic) IncrementalAlterConfigsResponseData(org.apache.kafka.common.message.IncrementalAlterConfigsResponseData) AccessControlEntryFilter(org.apache.kafka.common.acl.AccessControlEntryFilter) HashMap(java.util.HashMap) DescribeProducersRequest(org.apache.kafka.common.requests.DescribeProducersRequest) LeaderNotAvailableException(org.apache.kafka.common.errors.LeaderNotAvailableException) ApiError(org.apache.kafka.common.requests.ApiError) ConfigResource(org.apache.kafka.common.config.ConfigResource) OffsetOutOfRangeException(org.apache.kafka.common.errors.OffsetOutOfRangeException) MetadataRequest(org.apache.kafka.common.requests.MetadataRequest) AclBinding(org.apache.kafka.common.acl.AclBinding) OffsetDeleteResponseTopic(org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic) ClientQuotaEntity(org.apache.kafka.common.quota.ClientQuotaEntity) ApiVersion(org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion) OngoingTopicReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment) ConsumerPartitionAssignor(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor) Utils(org.apache.kafka.common.utils.Utils) Iterator(java.util.Iterator) Collections.emptySet(java.util.Collections.emptySet) DeleteAclsResponseData(org.apache.kafka.common.message.DeleteAclsResponseData) ApiKeys(org.apache.kafka.common.protocol.ApiKeys) TimeUnit(java.util.concurrent.TimeUnit) AlterPartitionReassignmentsResponseData(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData) DeleteTopicsResponse(org.apache.kafka.common.requests.DeleteTopicsResponse) MetadataResponseData(org.apache.kafka.common.message.MetadataResponseData) OffsetDeleteResponse(org.apache.kafka.common.requests.OffsetDeleteResponse) DescribeLogDirsResponse(org.apache.kafka.common.requests.DescribeLogDirsResponse) Timeout(org.junit.jupiter.api.Timeout) Collections(java.util.Collections) ApiException(org.apache.kafka.common.errors.ApiException) ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Node(org.apache.kafka.common.Node) ArrayList(java.util.ArrayList) Cluster(org.apache.kafka.common.Cluster) ListOffsetsResponseData(org.apache.kafka.common.message.ListOffsetsResponseData) ListOffsetsRequest(org.apache.kafka.common.requests.ListOffsetsRequest) TopicPartition(org.apache.kafka.common.TopicPartition) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) PartitionInfo(org.apache.kafka.common.PartitionInfo) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Example 5 with ListOffsetsResponse

use of org.apache.kafka.common.requests.ListOffsetsResponse in project kafka by apache.

the class KafkaAdminClientTest method testListOffsetsMetadataRetriableErrors.

@Test
public void testListOffsetsMetadataRetriableErrors() throws Exception {
    Node node0 = new Node(0, "localhost", 8120);
    Node node1 = new Node(1, "localhost", 8121);
    List<Node> nodes = Arrays.asList(node0, node1);
    List<PartitionInfo> pInfos = new ArrayList<>();
    pInfos.add(new PartitionInfo("foo", 0, node0, new Node[] { node0 }, new Node[] { node0 }));
    pInfos.add(new PartitionInfo("foo", 1, node1, new Node[] { node1 }, new Node[] { node1 }));
    final Cluster cluster = new Cluster("mockClusterId", nodes, pInfos, Collections.<String>emptySet(), Collections.<String>emptySet(), node0);
    final TopicPartition tp0 = new TopicPartition("foo", 0);
    final TopicPartition tp1 = new TopicPartition("foo", 1);
    try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) {
        env.kafkaClient().setNodeApiVersions(NodeApiVersions.create());
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.LEADER_NOT_AVAILABLE));
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.UNKNOWN_TOPIC_OR_PARTITION));
        env.kafkaClient().prepareResponse(prepareMetadataResponse(cluster, Errors.NONE));
        // listoffsets response from broker 0
        ListOffsetsTopicResponse t0 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp0, Errors.NONE, -1L, 345L, 543);
        ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(t0));
        env.kafkaClient().prepareResponseFrom(new ListOffsetsResponse(responseData), node0);
        // listoffsets response from broker 1
        ListOffsetsTopicResponse t1 = ListOffsetsResponse.singletonListOffsetsTopicResponse(tp1, Errors.NONE, -1L, 789L, 987);
        responseData = new ListOffsetsResponseData().setThrottleTimeMs(0).setTopics(Arrays.asList(t1));
        env.kafkaClient().prepareResponseFrom(new ListOffsetsResponse(responseData), node1);
        Map<TopicPartition, OffsetSpec> partitions = new HashMap<>();
        partitions.put(tp0, OffsetSpec.latest());
        partitions.put(tp1, OffsetSpec.latest());
        ListOffsetsResult result = env.adminClient().listOffsets(partitions);
        Map<TopicPartition, ListOffsetsResultInfo> offsets = result.all().get();
        assertFalse(offsets.isEmpty());
        assertEquals(345L, offsets.get(tp0).offset());
        assertEquals(543, offsets.get(tp0).leaderEpoch().get().intValue());
        assertEquals(-1L, offsets.get(tp0).timestamp());
        assertEquals(789L, offsets.get(tp1).offset());
        assertEquals(987, offsets.get(tp1).leaderEpoch().get().intValue());
        assertEquals(-1L, offsets.get(tp1).timestamp());
    }
}
Also used : ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) HashMap(java.util.HashMap) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Node(org.apache.kafka.common.Node) ArrayList(java.util.ArrayList) Cluster(org.apache.kafka.common.Cluster) ListOffsetsResponseData(org.apache.kafka.common.message.ListOffsetsResponseData) TopicPartition(org.apache.kafka.common.TopicPartition) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) PartitionInfo(org.apache.kafka.common.PartitionInfo) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Aggregations

ListOffsetsResponse (org.apache.kafka.common.requests.ListOffsetsResponse)19 TopicPartition (org.apache.kafka.common.TopicPartition)18 ListOffsetsResponseData (org.apache.kafka.common.message.ListOffsetsResponseData)18 ListOffsetsTopicResponse (org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse)18 HashMap (java.util.HashMap)14 Test (org.junit.jupiter.api.Test)14 Node (org.apache.kafka.common.Node)13 Cluster (org.apache.kafka.common.Cluster)11 PartitionInfo (org.apache.kafka.common.PartitionInfo)11 ArrayList (java.util.ArrayList)10 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)10 ListOffsetsResultInfo (org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo)9 ListOffsetsPartitionResponse (org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse)8 LinkedHashMap (java.util.LinkedHashMap)5 MetadataResponse (org.apache.kafka.common.requests.MetadataResponse)5 Map (java.util.Map)4 Errors (org.apache.kafka.common.protocol.Errors)4 List (java.util.List)3 OffsetAndTimestamp (org.apache.kafka.clients.consumer.OffsetAndTimestamp)3 InetSocketAddress (java.net.InetSocketAddress)2