Search in sources :

Example 1 with LATEST_SUPPORTED_VERSION

use of org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION in project kafka by apache.

the class StreamsPartitionAssignor method subscriptionUserData.

@Override
public ByteBuffer subscriptionUserData(final Set<String> topics) {
    // Adds the following information to subscription
    // 1. Client UUID (a unique id assigned to an instance of KafkaStreams)
    // 2. Map from task id to its overall lag
    // 3. Unique Field to ensure a rebalance when a thread rejoins by forcing the user data to be different
    handleRebalanceStart(topics);
    uniqueField++;
    final Set<String> currentNamedTopologies = taskManager.topologyMetadata().namedTopologiesView();
    // If using NamedTopologies, filter out any that are no longer recognized/have been removed
    final Map<TaskId, Long> taskOffsetSums = taskManager.topologyMetadata().hasNamedTopologies() ? filterMap(taskManager.getTaskOffsetSums(), t -> currentNamedTopologies.contains(t.getKey().topologyName())) : taskManager.getTaskOffsetSums();
    return new SubscriptionInfo(usedSubscriptionMetadataVersion, LATEST_SUPPORTED_VERSION, taskManager.processId(), userEndPoint, taskOffsetSums, uniqueField, assignmentErrorCode.get()).encode();
}
Also used : ClientUtils.fetchEndOffsetsFuture(org.apache.kafka.streams.processor.internals.ClientUtils.fetchEndOffsetsFuture) FallbackPriorTaskAssignor(org.apache.kafka.streams.processor.internals.assignment.FallbackPriorTaskAssignor) SortedSet(java.util.SortedSet) ConsumerGroupMetadata(org.apache.kafka.clients.consumer.ConsumerGroupMetadata) PriorityQueue(java.util.PriorityQueue) KafkaException(org.apache.kafka.common.KafkaException) StreamsException(org.apache.kafka.streams.errors.StreamsException) ClientUtils.fetchCommittedOffsets(org.apache.kafka.streams.processor.internals.ClientUtils.fetchCommittedOffsets) ByteBuffer(java.nio.ByteBuffer) UNKNOWN_OFFSET_SUM(org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo.UNKNOWN_OFFSET_SUM) Cluster(org.apache.kafka.common.Cluster) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LogContext(org.apache.kafka.common.utils.LogContext) Map(java.util.Map) MissingSourceTopicException(org.apache.kafka.streams.errors.MissingSourceTopicException) Consumer(org.apache.kafka.clients.consumer.Consumer) TopicPartition(org.apache.kafka.common.TopicPartition) Configurable(org.apache.kafka.common.Configurable) Time(org.apache.kafka.common.utils.Time) ReferenceContainer(org.apache.kafka.streams.processor.internals.assignment.ReferenceContainer) LATEST_SUPPORTED_VERSION(org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.LATEST_SUPPORTED_VERSION) Collection(java.util.Collection) Set(java.util.Set) KafkaFuture(org.apache.kafka.common.KafkaFuture) PartitionInfo(org.apache.kafka.common.PartitionInfo) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) AssignorConfiguration(org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration) Objects(java.util.Objects) ListOffsetsResultInfo(org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo) Utils.filterMap(org.apache.kafka.common.utils.Utils.filterMap) List(java.util.List) Node(org.apache.kafka.common.Node) Queue(java.util.Queue) SubscriptionInfo(org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo) TaskId(org.apache.kafka.streams.processor.TaskId) AssignmentConfigs(org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs) AssignmentInfo(org.apache.kafka.streams.processor.internals.assignment.AssignmentInfo) HostInfo(org.apache.kafka.streams.state.HostInfo) HashMap(java.util.HashMap) AssignmentListener(org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentListener) Supplier(java.util.function.Supplier) TreeSet(java.util.TreeSet) ArrayList(java.util.ArrayList) CopartitionedTopicsEnforcer(org.apache.kafka.streams.processor.internals.assignment.CopartitionedTopicsEnforcer) HashSet(java.util.HashSet) UNKNOWN(org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.UNKNOWN) StickyTaskAssignor(org.apache.kafka.streams.processor.internals.assignment.StickyTaskAssignor) Admin(org.apache.kafka.clients.admin.Admin) LinkedList(java.util.LinkedList) ConsumerPartitionAssignor(org.apache.kafka.clients.consumer.ConsumerPartitionAssignor) Utils(org.apache.kafka.common.utils.Utils) EARLIEST_PROBEABLE_VERSION(org.apache.kafka.streams.processor.internals.assignment.StreamsAssignmentProtocolVersions.EARLIEST_PROBEABLE_VERSION) TimeoutException(org.apache.kafka.common.errors.TimeoutException) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) TaskAssignmentException(org.apache.kafka.streams.errors.TaskAssignmentException) TopicsInfo(org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo) AssignorError(org.apache.kafka.streams.processor.internals.assignment.AssignorError) AtomicLong(java.util.concurrent.atomic.AtomicLong) UUID.randomUUID(java.util.UUID.randomUUID) TreeMap(java.util.TreeMap) ClientState(org.apache.kafka.streams.processor.internals.assignment.ClientState) TaskAssignor(org.apache.kafka.streams.processor.internals.assignment.TaskAssignor) Comparator(java.util.Comparator) Subtopology(org.apache.kafka.streams.processor.internals.TopologyMetadata.Subtopology) Collections(java.util.Collections) TaskId(org.apache.kafka.streams.processor.TaskId) AtomicLong(java.util.concurrent.atomic.AtomicLong) SubscriptionInfo(org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo)

Aggregations

ByteBuffer (java.nio.ByteBuffer)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 Comparator (java.util.Comparator)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 Iterator (java.util.Iterator)1 LinkedList (java.util.LinkedList)1 List (java.util.List)1 Map (java.util.Map)1 Objects (java.util.Objects)1 PriorityQueue (java.util.PriorityQueue)1 Queue (java.util.Queue)1 Set (java.util.Set)1 SortedSet (java.util.SortedSet)1 TreeMap (java.util.TreeMap)1 TreeSet (java.util.TreeSet)1 UUID (java.util.UUID)1 UUID.randomUUID (java.util.UUID.randomUUID)1