Search in sources :

Example 1 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class ListOffsetsRequest method getErrorResponse.

@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
    short versionId = version();
    short errorCode = Errors.forException(e).code();
    List<ListOffsetsTopicResponse> responses = new ArrayList<>();
    for (ListOffsetsTopic topic : data.topics()) {
        ListOffsetsTopicResponse topicResponse = new ListOffsetsTopicResponse().setName(topic.name());
        List<ListOffsetsPartitionResponse> partitions = new ArrayList<>();
        for (ListOffsetsPartition partition : topic.partitions()) {
            ListOffsetsPartitionResponse partitionResponse = new ListOffsetsPartitionResponse().setErrorCode(errorCode).setPartitionIndex(partition.partitionIndex());
            if (versionId == 0) {
                partitionResponse.setOldStyleOffsets(Collections.emptyList());
            } else {
                partitionResponse.setOffset(ListOffsetsResponse.UNKNOWN_OFFSET).setTimestamp(ListOffsetsResponse.UNKNOWN_TIMESTAMP);
            }
            partitions.add(partitionResponse);
        }
        topicResponse.setPartitions(partitions);
        responses.add(topicResponse);
    }
    ListOffsetsResponseData responseData = new ListOffsetsResponseData().setThrottleTimeMs(throttleTimeMs).setTopics(responses);
    return new ListOffsetsResponse(responseData);
}
Also used : ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) ListOffsetsTopic(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) ArrayList(java.util.ArrayList) ListOffsetsResponseData(org.apache.kafka.common.message.ListOffsetsResponseData)

Example 2 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class ListOffsetsRequest method toListOffsetsTopics.

public static List<ListOffsetsTopic> toListOffsetsTopics(Map<TopicPartition, ListOffsetsPartition> timestampsToSearch) {
    Map<String, ListOffsetsTopic> topics = new HashMap<>();
    for (Map.Entry<TopicPartition, ListOffsetsPartition> entry : timestampsToSearch.entrySet()) {
        TopicPartition tp = entry.getKey();
        ListOffsetsTopic topic = topics.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
        topic.partitions().add(entry.getValue());
    }
    return new ArrayList<>(topics.values());
}
Also used : ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) ListOffsetsTopic(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic) HashMap(java.util.HashMap) TopicPartition(org.apache.kafka.common.TopicPartition) ArrayList(java.util.ArrayList) HashMap(java.util.HashMap) Map(java.util.Map)

Example 3 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class KafkaAdminClient method getListOffsetsCalls.

// visible for benchmark
List<Call> getListOffsetsCalls(MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> context, Map<TopicPartition, OffsetSpec> topicPartitionOffsets, Map<TopicPartition, KafkaFutureImpl<ListOffsetsResultInfo>> futures) {
    MetadataResponse mr = context.response().orElseThrow(() -> new IllegalStateException("No Metadata response"));
    Cluster clusterSnapshot = mr.buildCluster();
    List<Call> calls = new ArrayList<>();
    // grouping topic partitions per leader
    Map<Node, Map<String, ListOffsetsTopic>> leaders = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetSpec> entry : topicPartitionOffsets.entrySet()) {
        OffsetSpec offsetSpec = entry.getValue();
        TopicPartition tp = entry.getKey();
        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
        long offsetQuery = getOffsetFromOffsetSpec(offsetSpec);
        // avoid sending listOffsets request for topics with errors
        if (!mr.errors().containsKey(tp.topic())) {
            Node node = clusterSnapshot.leaderFor(tp);
            if (node != null) {
                Map<String, ListOffsetsTopic> leadersOnNode = leaders.computeIfAbsent(node, k -> new HashMap<>());
                ListOffsetsTopic topic = leadersOnNode.computeIfAbsent(tp.topic(), k -> new ListOffsetsTopic().setName(tp.topic()));
                topic.partitions().add(new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offsetQuery));
            } else {
                future.completeExceptionally(Errors.LEADER_NOT_AVAILABLE.exception());
            }
        } else {
            future.completeExceptionally(mr.errors().get(tp.topic()).exception());
        }
    }
    for (final Map.Entry<Node, Map<String, ListOffsetsTopic>> entry : leaders.entrySet()) {
        final int brokerId = entry.getKey().id();
        calls.add(new Call("listOffsets on broker " + brokerId, context.deadline(), new ConstantNodeIdProvider(brokerId)) {

            final List<ListOffsetsTopic> partitionsToQuery = new ArrayList<>(entry.getValue().values());

            private boolean supportsMaxTimestamp = partitionsToQuery.stream().flatMap(t -> t.partitions().stream()).anyMatch(p -> p.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP);

            @Override
            ListOffsetsRequest.Builder createRequest(int timeoutMs) {
                return ListOffsetsRequest.Builder.forConsumer(true, context.options().isolationLevel(), supportsMaxTimestamp).setTargetTimes(partitionsToQuery);
            }

            @Override
            void handleResponse(AbstractResponse abstractResponse) {
                ListOffsetsResponse response = (ListOffsetsResponse) abstractResponse;
                Map<TopicPartition, OffsetSpec> retryTopicPartitionOffsets = new HashMap<>();
                for (ListOffsetsTopicResponse topic : response.topics()) {
                    for (ListOffsetsPartitionResponse partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        Errors error = Errors.forCode(partition.errorCode());
                        OffsetSpec offsetRequestSpec = topicPartitionOffsets.get(tp);
                        if (offsetRequestSpec == null) {
                            log.warn("Server response mentioned unknown topic partition {}", tp);
                        } else if (MetadataOperationContext.shouldRefreshMetadata(error)) {
                            retryTopicPartitionOffsets.put(tp, offsetRequestSpec);
                        } else if (error == Errors.NONE) {
                            Optional<Integer> leaderEpoch = (partition.leaderEpoch() == ListOffsetsResponse.UNKNOWN_EPOCH) ? Optional.empty() : Optional.of(partition.leaderEpoch());
                            future.complete(new ListOffsetsResultInfo(partition.offset(), partition.timestamp(), leaderEpoch));
                        } else {
                            future.completeExceptionally(error.exception());
                        }
                    }
                }
                if (retryTopicPartitionOffsets.isEmpty()) {
                    // The server should send back a response for every topic partition. But do a sanity check anyway.
                    for (ListOffsetsTopic topic : partitionsToQuery) {
                        for (ListOffsetsPartition partition : topic.partitions()) {
                            TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                            ApiException error = new ApiException("The response from broker " + brokerId + " did not contain a result for topic partition " + tp);
                            futures.get(tp).completeExceptionally(error);
                        }
                    }
                } else {
                    Set<String> retryTopics = retryTopicPartitionOffsets.keySet().stream().map(TopicPartition::topic).collect(Collectors.toSet());
                    MetadataOperationContext<ListOffsetsResultInfo, ListOffsetsOptions> retryContext = new MetadataOperationContext<>(retryTopics, context.options(), context.deadline(), futures);
                    rescheduleMetadataTask(retryContext, () -> getListOffsetsCalls(retryContext, retryTopicPartitionOffsets, futures));
                }
            }

            @Override
            void handleFailure(Throwable throwable) {
                for (ListOffsetsTopic topic : entry.getValue().values()) {
                    for (ListOffsetsPartition partition : topic.partitions()) {
                        TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex());
                        KafkaFutureImpl<ListOffsetsResultInfo> future = futures.get(tp);
                        future.completeExceptionally(throwable);
                    }
                }
            }

            @Override
            boolean handleUnsupportedVersionException(UnsupportedVersionException exception) {
                if (supportsMaxTimestamp) {
                    supportsMaxTimestamp = false;
                    // fail any unsupported futures and remove partitions from the downgraded retry
                    Iterator<ListOffsetsTopic> topicIterator = partitionsToQuery.iterator();
                    while (topicIterator.hasNext()) {
                        ListOffsetsTopic topic = topicIterator.next();
                        Iterator<ListOffsetsPartition> partitionIterator = topic.partitions().iterator();
                        while (partitionIterator.hasNext()) {
                            ListOffsetsPartition partition = partitionIterator.next();
                            if (partition.timestamp() == ListOffsetsRequest.MAX_TIMESTAMP) {
                                futures.get(new TopicPartition(topic.name(), partition.partitionIndex())).completeExceptionally(new UnsupportedVersionException("Broker " + brokerId + " does not support MAX_TIMESTAMP offset spec"));
                                partitionIterator.remove();
                            }
                        }
                        if (topic.partitions().isEmpty()) {
                            topicIterator.remove();
                        }
                    }
                    return !partitionsToQuery.isEmpty();
                }
                return false;
            }
        });
    }
    return calls;
}
Also used : ListGroupsResponse(org.apache.kafka.common.requests.ListGroupsResponse) ThrottlingQuotaExceededException(org.apache.kafka.common.errors.ThrottlingQuotaExceededException) ListOffsetsRequest(org.apache.kafka.common.requests.ListOffsetsRequest) UserName(org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData.UserName) ListPartitionReassignmentsTopics(org.apache.kafka.common.message.ListPartitionReassignmentsRequestData.ListPartitionReassignmentsTopics) ClientQuotaFilter(org.apache.kafka.common.quota.ClientQuotaFilter) KafkaException(org.apache.kafka.common.KafkaException) AppInfoParser(org.apache.kafka.common.utils.AppInfoParser) ClientUtils(org.apache.kafka.clients.ClientUtils) DescribeClientQuotasRequest(org.apache.kafka.common.requests.DescribeClientQuotasRequest) ListGroupsRequestData(org.apache.kafka.common.message.ListGroupsRequestData) AclCreationResult(org.apache.kafka.common.message.CreateAclsResponseData.AclCreationResult) Duration(java.time.Duration) Map(java.util.Map) TopicNameCollection(org.apache.kafka.common.TopicCollection.TopicNameCollection) UpdateFeaturesRequestData(org.apache.kafka.common.message.UpdateFeaturesRequestData) Utils.closeQuietly(org.apache.kafka.common.utils.Utils.closeQuietly) DeleteAclsFilter(org.apache.kafka.common.message.DeleteAclsRequestData.DeleteAclsFilter) CreateAclsRequest(org.apache.kafka.common.requests.CreateAclsRequest) DescribeConfigsRequestData(org.apache.kafka.common.message.DescribeConfigsRequestData) DescribableLogDirTopic(org.apache.kafka.common.message.DescribeLogDirsRequestData.DescribableLogDirTopic) CommonClientConfigs(org.apache.kafka.clients.CommonClientConfigs) Sensor(org.apache.kafka.common.metrics.Sensor) DescribeLogDirsResponseData(org.apache.kafka.common.message.DescribeLogDirsResponseData) ClientQuotaAlteration(org.apache.kafka.common.quota.ClientQuotaAlteration) AlterReplicaLogDirTopicResult(org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirTopicResult) DeleteTopicsRequestData(org.apache.kafka.common.message.DeleteTopicsRequestData) AlterReplicaLogDirsRequest(org.apache.kafka.common.requests.AlterReplicaLogDirsRequest) ListGroupsRequest(org.apache.kafka.common.requests.ListGroupsRequest) Metrics(org.apache.kafka.common.metrics.Metrics) Stream(java.util.stream.Stream) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Errors(org.apache.kafka.common.protocol.Errors) Node(org.apache.kafka.common.Node) UnknownTopicOrPartitionException(org.apache.kafka.common.errors.UnknownTopicOrPartitionException) UnacceptableCredentialException(org.apache.kafka.common.errors.UnacceptableCredentialException) AuthenticationException(org.apache.kafka.common.errors.AuthenticationException) DeleteTopicState(org.apache.kafka.common.message.DeleteTopicsRequestData.DeleteTopicState) SupportedFeatureKey(org.apache.kafka.common.message.ApiVersionsResponseData.SupportedFeatureKey) DeleteRecordsTopic(org.apache.kafka.common.message.DeleteRecordsRequestData.DeleteRecordsTopic) AbstractResponse(org.apache.kafka.common.requests.AbstractResponse) AlterUserScramCredentialsResponse(org.apache.kafka.common.requests.AlterUserScramCredentialsResponse) DescribeClusterResponse(org.apache.kafka.common.requests.DescribeClusterResponse) DelegationToken(org.apache.kafka.common.security.token.delegation.DelegationToken) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) Supplier(java.util.function.Supplier) DescribeAclsResponse(org.apache.kafka.common.requests.DescribeAclsResponse) AlterReplicaLogDirsResponse(org.apache.kafka.common.requests.AlterReplicaLogDirsResponse) AlterConsumerGroupOffsetsHandler(org.apache.kafka.clients.admin.internals.AlterConsumerGroupOffsetsHandler) UpdateFeaturesRequest(org.apache.kafka.common.requests.UpdateFeaturesRequest) KafkaStorageException(org.apache.kafka.common.errors.KafkaStorageException) AlterPartitionReassignmentsRequest(org.apache.kafka.common.requests.AlterPartitionReassignmentsRequest) DeleteAclsMatchingAcl(org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl) DescribeLogDirsRequestData(org.apache.kafka.common.message.DescribeLogDirsRequestData) ApiVersions(org.apache.kafka.clients.ApiVersions) TopicMetadataAndConfig(org.apache.kafka.clients.admin.CreateTopicsResult.TopicMetadataAndConfig) IncrementalAlterConfigsRequest(org.apache.kafka.common.requests.IncrementalAlterConfigsRequest) AtomicLong(java.util.concurrent.atomic.AtomicLong) CreatableTopicCollection(org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopicCollection) ReassignableTopic(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignableTopic) TreeMap(java.util.TreeMap) IncrementalAlterConfigsResponse(org.apache.kafka.common.requests.IncrementalAlterConfigsResponse) UpdatableFeatureResult(org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResult) CreatePartitionsAssignment(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsAssignment) ElectLeadersRequest(org.apache.kafka.common.requests.ElectLeadersRequest) FilterResults(org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults) DescribeUserScramCredentialsResponse(org.apache.kafka.common.requests.DescribeUserScramCredentialsResponse) AdminApiFuture(org.apache.kafka.clients.admin.internals.AdminApiFuture) AbstractRequest(org.apache.kafka.common.requests.AbstractRequest) ChannelBuilder(org.apache.kafka.common.network.ChannelBuilder) MemberIdentity(org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity) TokenInformation(org.apache.kafka.common.security.token.delegation.TokenInformation) CreateDelegationTokenRequestData(org.apache.kafka.common.message.CreateDelegationTokenRequestData) CreateAclsRequestData(org.apache.kafka.common.message.CreateAclsRequestData) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) KafkaClient(org.apache.kafka.clients.KafkaClient) DescribeLogDirsRequest(org.apache.kafka.common.requests.DescribeLogDirsRequest) DescribeConfigsResponseData(org.apache.kafka.common.message.DescribeConfigsResponseData) TopicIdCollection(org.apache.kafka.common.TopicCollection.TopicIdCollection) ListOffsetsTopic(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic) TopicPartition(org.apache.kafka.common.TopicPartition) Time(org.apache.kafka.common.utils.Time) Collection(java.util.Collection) AlterReplicaLogDir(org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDir) CreateAclsResponse(org.apache.kafka.common.requests.CreateAclsResponse) DescribeClusterRequest(org.apache.kafka.common.requests.DescribeClusterRequest) DescribeUserScramCredentialsResponseData(org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData) Collectors(java.util.stream.Collectors) DescribeConfigsRequest(org.apache.kafka.common.requests.DescribeConfigsRequest) Objects(java.util.Objects) CreatePartitionsResponse(org.apache.kafka.common.requests.CreatePartitionsResponse) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) UnregisterBrokerRequestData(org.apache.kafka.common.message.UnregisterBrokerRequestData) AlterUserScramCredentialsRequest(org.apache.kafka.common.requests.AlterUserScramCredentialsRequest) RenewDelegationTokenRequestData(org.apache.kafka.common.message.RenewDelegationTokenRequestData) ExpireDelegationTokenRequest(org.apache.kafka.common.requests.ExpireDelegationTokenRequest) Uuid(org.apache.kafka.common.Uuid) DeleteTopicsRequest(org.apache.kafka.common.requests.DeleteTopicsRequest) ReassignablePartitionResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse) DeleteRecordsRequestData(org.apache.kafka.common.message.DeleteRecordsRequestData) FinalizedFeatureKey(org.apache.kafka.common.message.ApiVersionsResponseData.FinalizedFeatureKey) DeleteAclsResponse(org.apache.kafka.common.requests.DeleteAclsResponse) DeleteAclsFilterResult(org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult) AlterPartitionReassignmentsResponse(org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse) DescribeConfigsResponse(org.apache.kafka.common.requests.DescribeConfigsResponse) Function(java.util.function.Function) UnknownServerException(org.apache.kafka.common.errors.UnknownServerException) DeleteRecordsResponseData(org.apache.kafka.common.message.DeleteRecordsResponseData) DescribeUserScramCredentialsRequest(org.apache.kafka.common.requests.DescribeUserScramCredentialsRequest) HashSet(java.util.HashSet) CreatableTopicConfigs(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicConfigs) TopicCollection(org.apache.kafka.common.TopicCollection) OngoingPartitionReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment) LinkedList(java.util.LinkedList) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Logger(org.slf4j.Logger) AllBrokersStrategy(org.apache.kafka.clients.admin.internals.AllBrokersStrategy) AlterConfigsResponse(org.apache.kafka.common.requests.AlterConfigsResponse) CreateTopicsResponse(org.apache.kafka.common.requests.CreateTopicsResponse) ConfigException(org.apache.kafka.common.config.ConfigException) DeleteRecordsPartition(org.apache.kafka.common.message.DeleteRecordsRequestData.DeleteRecordsPartition) KafkaThread(org.apache.kafka.common.utils.KafkaThread) AlterConfigsRequest(org.apache.kafka.common.requests.AlterConfigsRequest) RenewDelegationTokenRequest(org.apache.kafka.common.requests.RenewDelegationTokenRequest) DefaultHostResolver(org.apache.kafka.clients.DefaultHostResolver) DisconnectException(org.apache.kafka.common.errors.DisconnectException) InvalidRequestException(org.apache.kafka.common.errors.InvalidRequestException) KafkaPrincipal(org.apache.kafka.common.security.auth.KafkaPrincipal) Comparator(java.util.Comparator) ExpireDelegationTokenResponse(org.apache.kafka.common.requests.ExpireDelegationTokenResponse) ReplicaLogDirInfo(org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo) Arrays(java.util.Arrays) ApiVersionsRequest(org.apache.kafka.common.requests.ApiVersionsRequest) DescribeClusterRequestData(org.apache.kafka.common.message.DescribeClusterRequestData) CreatePartitionsTopicCollection(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopicCollection) DeleteRecordsTopicResult(org.apache.kafka.common.message.DeleteRecordsResponseData.DeleteRecordsTopicResult) Cluster(org.apache.kafka.common.Cluster) AbortTransactionHandler(org.apache.kafka.clients.admin.internals.AbortTransactionHandler) AdminApiHandler(org.apache.kafka.clients.admin.internals.AdminApiHandler) UnregisterBrokerRequest(org.apache.kafka.common.requests.UnregisterBrokerRequest) CreatePartitionsTopic(org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic) RemoveMembersFromConsumerGroupHandler(org.apache.kafka.clients.admin.internals.RemoveMembersFromConsumerGroupHandler) ApiVersionsResponse(org.apache.kafka.common.requests.ApiVersionsResponse) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) LogContext(org.apache.kafka.common.utils.LogContext) ListPartitionReassignmentsRequestData(org.apache.kafka.common.message.ListPartitionReassignmentsRequestData) CreatePartitionsRequestData(org.apache.kafka.common.message.CreatePartitionsRequestData) TimestampSpec(org.apache.kafka.clients.admin.OffsetSpec.TimestampSpec) ListTransactionsHandler(org.apache.kafka.clients.admin.internals.ListTransactionsHandler) DeleteAclsRequest(org.apache.kafka.common.requests.DeleteAclsRequest) InvalidTopicException(org.apache.kafka.common.errors.InvalidTopicException) Set(java.util.Set) PartitionInfo(org.apache.kafka.common.PartitionInfo) DeleteRecordsRequest(org.apache.kafka.common.requests.DeleteRecordsRequest) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) CoordinatorKey(org.apache.kafka.clients.admin.internals.CoordinatorKey) MetricsReporter(org.apache.kafka.common.metrics.MetricsReporter) ConsumerGroupState(org.apache.kafka.common.ConsumerGroupState) InvalidKeyException(java.security.InvalidKeyException) CreatableTopicResult(org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult) MetadataResponse(org.apache.kafka.common.requests.MetadataResponse) AlterUserScramCredentialsRequestData(org.apache.kafka.common.message.AlterUserScramCredentialsRequestData) CreatableRenewers(org.apache.kafka.common.message.CreateDelegationTokenRequestData.CreatableRenewers) Selector(org.apache.kafka.common.network.Selector) KafkaMetricsContext(org.apache.kafka.common.metrics.KafkaMetricsContext) RenewDelegationTokenResponse(org.apache.kafka.common.requests.RenewDelegationTokenResponse) RetriableException(org.apache.kafka.common.errors.RetriableException) ExpireDelegationTokenRequestData(org.apache.kafka.common.message.ExpireDelegationTokenRequestData) ArrayList(java.util.ArrayList) MetricsContext(org.apache.kafka.common.metrics.MetricsContext) ElectionType(org.apache.kafka.common.ElectionType) NetworkClient(org.apache.kafka.clients.NetworkClient) InterfaceStability(org.apache.kafka.common.annotation.InterfaceStability) AlterPartitionReassignmentsRequestData(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) MetadataOperationContext(org.apache.kafka.clients.admin.internals.MetadataOperationContext) ListGroupsResponseData(org.apache.kafka.common.message.ListGroupsResponseData) TopicPartitionInfo(org.apache.kafka.common.TopicPartitionInfo) ScramFormatter(org.apache.kafka.common.security.scram.internals.ScramFormatter) AlterClientQuotasResponse(org.apache.kafka.common.requests.AlterClientQuotasResponse) AclCreation(org.apache.kafka.common.message.CreateAclsRequestData.AclCreation) TopicPartitionReplica(org.apache.kafka.common.TopicPartitionReplica) DeleteRecordsResponse(org.apache.kafka.common.requests.DeleteRecordsResponse) ReassignablePartition(org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData.ReassignablePartition) AclOperation(org.apache.kafka.common.acl.AclOperation) AlterReplicaLogDirsRequestData(org.apache.kafka.common.message.AlterReplicaLogDirsRequestData) UnsupportedSaslMechanismException(org.apache.kafka.common.errors.UnsupportedSaslMechanismException) CreateTopicsRequestData(org.apache.kafka.common.message.CreateTopicsRequestData) DescribeUserScramCredentialsRequestData(org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException) MetadataRequestData(org.apache.kafka.common.message.MetadataRequestData) DescribeDelegationTokenResponse(org.apache.kafka.common.requests.DescribeDelegationTokenResponse) AdminApiDriver(org.apache.kafka.clients.admin.internals.AdminApiDriver) AlterClientQuotasRequest(org.apache.kafka.common.requests.AlterClientQuotasRequest) CreatePartitionsTopicResult(org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult) SimpleAdminApiFuture(org.apache.kafka.clients.admin.internals.AdminApiFuture.SimpleAdminApiFuture) AlterReplicaLogDirPartitionResult(org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirPartitionResult) ListPartitionReassignmentsResponse(org.apache.kafka.common.requests.ListPartitionReassignmentsResponse) ClientRequest(org.apache.kafka.clients.ClientRequest) DescribeClientQuotasResponse(org.apache.kafka.common.requests.DescribeClientQuotasResponse) UpdateFeaturesResponse(org.apache.kafka.common.requests.UpdateFeaturesResponse) ReassignableTopicResponse(org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse) DeleteAclsRequestData(org.apache.kafka.common.message.DeleteAclsRequestData) UnregisterBrokerResponse(org.apache.kafka.common.requests.UnregisterBrokerResponse) CreatePartitionsRequest(org.apache.kafka.common.requests.CreatePartitionsRequest) MetadataRequest.convertToMetadataRequestTopic(org.apache.kafka.common.requests.MetadataRequest.convertToMetadataRequestTopic) Metric(org.apache.kafka.common.Metric) MetricName(org.apache.kafka.common.MetricName) AdminMetadataManager(org.apache.kafka.clients.admin.internals.AdminMetadataManager) DescribeAclsRequest(org.apache.kafka.common.requests.DescribeAclsRequest) Predicate(java.util.function.Predicate) MetricConfig(org.apache.kafka.common.metrics.MetricConfig) KafkaFuture(org.apache.kafka.common.KafkaFuture) InetSocketAddress(java.net.InetSocketAddress) ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) List(java.util.List) DeletableTopicResult(org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult) Optional(java.util.Optional) ClientResponse(org.apache.kafka.clients.ClientResponse) ConsumerProtocol(org.apache.kafka.clients.consumer.internals.ConsumerProtocol) ListConsumerGroupOffsetsHandler(org.apache.kafka.clients.admin.internals.ListConsumerGroupOffsetsHandler) MetadataRequest.convertTopicIdsToMetadataRequestTopic(org.apache.kafka.common.requests.MetadataRequest.convertTopicIdsToMetadataRequestTopic) CreateTopicsRequest(org.apache.kafka.common.requests.CreateTopicsRequest) ElectLeadersResponse(org.apache.kafka.common.requests.ElectLeadersResponse) StaleMetadataException(org.apache.kafka.clients.StaleMetadataException) AclBindingFilter(org.apache.kafka.common.acl.AclBindingFilter) HashMap(java.util.HashMap) DescribeConsumerGroupsHandler(org.apache.kafka.clients.admin.internals.DescribeConsumerGroupsHandler) ApiError(org.apache.kafka.common.requests.ApiError) ConfigResource(org.apache.kafka.common.config.ConfigResource) CreateDelegationTokenResponse(org.apache.kafka.common.requests.CreateDelegationTokenResponse) CreateDelegationTokenResponseData(org.apache.kafka.common.message.CreateDelegationTokenResponseData) DeleteConsumerGroupsHandler(org.apache.kafka.clients.admin.internals.DeleteConsumerGroupsHandler) MetadataRequest(org.apache.kafka.common.requests.MetadataRequest) AclBinding(org.apache.kafka.common.acl.AclBinding) ClientQuotaEntity(org.apache.kafka.common.quota.ClientQuotaEntity) DescribeProducersHandler(org.apache.kafka.clients.admin.internals.DescribeProducersHandler) OngoingTopicReassignment(org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment) ListPartitionReassignmentsRequest(org.apache.kafka.common.requests.ListPartitionReassignmentsRequest) DeleteConsumerGroupOffsetsHandler(org.apache.kafka.clients.admin.internals.DeleteConsumerGroupOffsetsHandler) AlterReplicaLogDirTopic(org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDirTopic) JmxReporter(org.apache.kafka.common.metrics.JmxReporter) Utils(org.apache.kafka.common.utils.Utils) DescribeTransactionsHandler(org.apache.kafka.clients.admin.internals.DescribeTransactionsHandler) ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) Iterator(java.util.Iterator) FilterResult(org.apache.kafka.clients.admin.DeleteAclsResult.FilterResult) DeleteAclsResponseData(org.apache.kafka.common.message.DeleteAclsResponseData) TimeUnit(java.util.concurrent.TimeUnit) DescribeDelegationTokenRequest(org.apache.kafka.common.requests.DescribeDelegationTokenRequest) HostResolver(org.apache.kafka.clients.HostResolver) DeleteTopicsResponse(org.apache.kafka.common.requests.DeleteTopicsResponse) DescribeLogDirsResponse(org.apache.kafka.common.requests.DescribeLogDirsResponse) Collections(java.util.Collections) ApiException(org.apache.kafka.common.errors.ApiException) CreateDelegationTokenRequest(org.apache.kafka.common.requests.CreateDelegationTokenRequest) HashSet(java.util.HashSet) Set(java.util.Set) ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) HashMap(java.util.HashMap) ListOffsetsTopicResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse) Node(org.apache.kafka.common.Node) ChannelBuilder(org.apache.kafka.common.network.ChannelBuilder) ListOffsetsPartitionResponse(org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse) ArrayList(java.util.ArrayList) ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) ListOffsetsTopic(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic) MetadataResponse(org.apache.kafka.common.requests.MetadataResponse) Iterator(java.util.Iterator) Optional(java.util.Optional) AbstractResponse(org.apache.kafka.common.requests.AbstractResponse) Cluster(org.apache.kafka.common.Cluster) MetadataOperationContext(org.apache.kafka.clients.admin.internals.MetadataOperationContext) KafkaFutureImpl(org.apache.kafka.common.internals.KafkaFutureImpl) Errors(org.apache.kafka.common.protocol.Errors) TopicPartition(org.apache.kafka.common.TopicPartition) ListOffsetsResponse(org.apache.kafka.common.requests.ListOffsetsResponse) Map(java.util.Map) TreeMap(java.util.TreeMap) HashMap(java.util.HashMap) ApiException(org.apache.kafka.common.errors.ApiException) UnsupportedVersionException(org.apache.kafka.common.errors.UnsupportedVersionException)

Example 4 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class Fetcher method resetOffsetsAsync.

private void resetOffsetsAsync(Map<TopicPartition, Long> partitionResetTimestamps) {
    Map<Node, Map<TopicPartition, ListOffsetsPartition>> timestampsToSearchByNode = groupListOffsetRequests(partitionResetTimestamps, new HashSet<>());
    for (Map.Entry<Node, Map<TopicPartition, ListOffsetsPartition>> entry : timestampsToSearchByNode.entrySet()) {
        Node node = entry.getKey();
        final Map<TopicPartition, ListOffsetsPartition> resetTimestamps = entry.getValue();
        subscriptions.setNextAllowedRetry(resetTimestamps.keySet(), time.milliseconds() + requestTimeoutMs);
        RequestFuture<ListOffsetResult> future = sendListOffsetRequest(node, resetTimestamps, false);
        future.addListener(new RequestFutureListener<ListOffsetResult>() {

            @Override
            public void onSuccess(ListOffsetResult result) {
                if (!result.partitionsToRetry.isEmpty()) {
                    subscriptions.requestFailed(result.partitionsToRetry, time.milliseconds() + retryBackoffMs);
                    metadata.requestUpdate();
                }
                for (Map.Entry<TopicPartition, ListOffsetData> fetchedOffset : result.fetchedOffsets.entrySet()) {
                    TopicPartition partition = fetchedOffset.getKey();
                    ListOffsetData offsetData = fetchedOffset.getValue();
                    ListOffsetsPartition requestedReset = resetTimestamps.get(partition);
                    resetOffsetIfNeeded(partition, timestampToOffsetResetStrategy(requestedReset.timestamp()), offsetData);
                }
            }

            @Override
            public void onFailure(RuntimeException e) {
                subscriptions.requestFailed(resetTimestamps.keySet(), time.milliseconds() + retryBackoffMs);
                metadata.requestUpdate();
                if (!(e instanceof RetriableException) && !cachedListOffsetsException.compareAndSet(null, e))
                    log.error("Discarding error in ListOffsetResponse because another error is pending", e);
            }
        });
    }
}
Also used : Node(org.apache.kafka.common.Node) ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) TopicPartition(org.apache.kafka.common.TopicPartition) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) RetriableException(org.apache.kafka.common.errors.RetriableException)

Example 5 with ListOffsetsPartition

use of org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition in project kafka by apache.

the class Fetcher method groupListOffsetRequests.

/**
 * Groups timestamps to search by node for topic partitions in `timestampsToSearch` that have
 * leaders available. Topic partitions from `timestampsToSearch` that do not have their leader
 * available are added to `partitionsToRetry`
 * @param timestampsToSearch The mapping from partitions ot the target timestamps
 * @param partitionsToRetry A set of topic partitions that will be extended with partitions
 *                          that need metadata update or re-connect to the leader.
 */
private Map<Node, Map<TopicPartition, ListOffsetsPartition>> groupListOffsetRequests(Map<TopicPartition, Long> timestampsToSearch, Set<TopicPartition> partitionsToRetry) {
    final Map<TopicPartition, ListOffsetsPartition> partitionDataMap = new HashMap<>();
    for (Map.Entry<TopicPartition, Long> entry : timestampsToSearch.entrySet()) {
        TopicPartition tp = entry.getKey();
        Long offset = entry.getValue();
        Metadata.LeaderAndEpoch leaderAndEpoch = metadata.currentLeader(tp);
        if (!leaderAndEpoch.leader.isPresent()) {
            log.debug("Leader for partition {} is unknown for fetching offset {}", tp, offset);
            metadata.requestUpdate();
            partitionsToRetry.add(tp);
        } else {
            Node leader = leaderAndEpoch.leader.get();
            if (client.isUnavailable(leader)) {
                client.maybeThrowAuthFailure(leader);
                // The connection has failed and we need to await the backoff period before we can
                // try again. No need to request a metadata update since the disconnect will have
                // done so already.
                log.debug("Leader {} for partition {} is unavailable for fetching offset until reconnect backoff expires", leader, tp);
                partitionsToRetry.add(tp);
            } else {
                int currentLeaderEpoch = leaderAndEpoch.epoch.orElse(ListOffsetsResponse.UNKNOWN_EPOCH);
                partitionDataMap.put(tp, new ListOffsetsPartition().setPartitionIndex(tp.partition()).setTimestamp(offset).setCurrentLeaderEpoch(currentLeaderEpoch));
            }
        }
    }
    return regroupPartitionMapByNode(partitionDataMap);
}
Also used : ListOffsetsPartition(org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) TopicPartition(org.apache.kafka.common.TopicPartition) Node(org.apache.kafka.common.Node) OffsetAndMetadata(org.apache.kafka.clients.consumer.OffsetAndMetadata) Metadata(org.apache.kafka.clients.Metadata) Map(java.util.Map) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap)

Aggregations

ListOffsetsPartition (org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition)14 ListOffsetsTopic (org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic)11 TopicPartition (org.apache.kafka.common.TopicPartition)7 Test (org.junit.jupiter.api.Test)7 HashMap (java.util.HashMap)5 Node (org.apache.kafka.common.Node)5 ListOffsetsPartitionResponse (org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse)5 ListOffsetsTopicResponse (org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse)5 Map (java.util.Map)4 ListOffsetsResponseData (org.apache.kafka.common.message.ListOffsetsResponseData)4 ArrayList (java.util.ArrayList)3 ListOffsetsRequest (org.apache.kafka.common.requests.ListOffsetsRequest)3 LinkedHashMap (java.util.LinkedHashMap)2 OffsetAndMetadata (org.apache.kafka.clients.consumer.OffsetAndMetadata)2 RetriableException (org.apache.kafka.common.errors.RetriableException)2 InetSocketAddress (java.net.InetSocketAddress)1 InvalidKeyException (java.security.InvalidKeyException)1 NoSuchAlgorithmException (java.security.NoSuchAlgorithmException)1 Duration (java.time.Duration)1 Arrays (java.util.Arrays)1