Examples with Subscription org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription used on opensource projects

Example 36 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class ConsumerProtocol method deserializeSubscription.

public static Subscription deserializeSubscription(final ByteBuffer buffer, short version) {
    version = checkSubscriptionVersion(version);
    try {
        ConsumerProtocolSubscription data = new ConsumerProtocolSubscription(new ByteBufferAccessor(buffer), version);
        List<TopicPartition> ownedPartitions = new ArrayList<>();
        for (ConsumerProtocolSubscription.TopicPartition tp : data.ownedPartitions()) {
            for (Integer partition : tp.partitions()) {
                ownedPartitions.add(new TopicPartition(tp.topic(), partition));
            }
        }
        return new Subscription(data.topics(), data.userData() != null ? data.userData().duplicate() : null, ownedPartitions);
    } catch (BufferUnderflowException e) {
        throw new SchemaException("Buffer underflow while parsing consumer protocol's subscription", e);
    }
}

Example 37 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class ConsumerCoordinator method onLeaderElected.

@Override
protected Map<String, ByteBuffer> onLeaderElected(String leaderId, String assignmentStrategy, List<JoinGroupResponseData.JoinGroupResponseMember> allSubscriptions, boolean skipAssignment) {
    ConsumerPartitionAssignor assignor = lookupAssignor(assignmentStrategy);
    if (assignor == null)
        throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
    String assignorName = assignor.name();
    Set<String> allSubscribedTopics = new HashSet<>();
    Map<String, Subscription> subscriptions = new HashMap<>();
    // collect all the owned partitions
    Map<String, List<TopicPartition>> ownedPartitions = new HashMap<>();
    for (JoinGroupResponseData.JoinGroupResponseMember memberSubscription : allSubscriptions) {
        Subscription subscription = ConsumerProtocol.deserializeSubscription(ByteBuffer.wrap(memberSubscription.metadata()));
        subscription.setGroupInstanceId(Optional.ofNullable(memberSubscription.groupInstanceId()));
        subscriptions.put(memberSubscription.memberId(), subscription);
        allSubscribedTopics.addAll(subscription.topics());
        ownedPartitions.put(memberSubscription.memberId(), subscription.ownedPartitions());
    }
    // the leader will begin watching for changes to any of the topics the group is interested in,
    // which ensures that all metadata changes will eventually be seen
    updateGroupSubscription(allSubscribedTopics);
    isLeader = true;
    if (skipAssignment) {
        log.info("Skipped assignment for returning static leader at generation {}. The static leader " + "will continue with its existing assignment.", generation().generationId);
        assignmentSnapshot = metadataSnapshot;
        return Collections.emptyMap();
    }
    log.debug("Performing assignment using strategy {} with subscriptions {}", assignorName, subscriptions);
    Map<String, Assignment> assignments = assignor.assign(metadata.fetch(), new GroupSubscription(subscriptions)).groupAssignment();
    // the "generation" of ownedPartition inside the assignor
    if (protocol == RebalanceProtocol.COOPERATIVE && !assignorName.equals(COOPERATIVE_STICKY_ASSIGNOR_NAME)) {
        validateCooperativeAssignment(ownedPartitions, assignments);
    }
    maybeUpdateGroupSubscription(assignorName, assignments, allSubscribedTopics);
    // metadataSnapshot could be updated when the subscription is updated therefore
    // we must take the assignment snapshot after.
    assignmentSnapshot = metadataSnapshot;
    log.info("Finished assignment for group at generation {}: {}", generation().generationId, assignments);
    Map<String, ByteBuffer> groupAssignment = new HashMap<>();
    for (Map.Entry<String, Assignment> assignmentEntry : assignments.entrySet()) {
        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(assignmentEntry.getValue());
        groupAssignment.put(assignmentEntry.getKey(), buffer);
    }
    return groupAssignment;
}

Example 38 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class ConsumerCoordinator method metadata.

@Override
protected JoinGroupRequestData.JoinGroupRequestProtocolCollection metadata() {
    log.debug("Joining group with current subscription: {}", subscriptions.subscription());
    this.joinedSubscription = subscriptions.subscription();
    JoinGroupRequestData.JoinGroupRequestProtocolCollection protocolSet = new JoinGroupRequestData.JoinGroupRequestProtocolCollection();
    List<String> topics = new ArrayList<>(joinedSubscription);
    for (ConsumerPartitionAssignor assignor : assignors) {
        Subscription subscription = new Subscription(topics, assignor.subscriptionUserData(joinedSubscription), subscriptions.assignedPartitionsList());
        ByteBuffer metadata = ConsumerProtocol.serializeSubscription(subscription);
        protocolSet.add(new JoinGroupRequestData.JoinGroupRequestProtocol().setName(assignor.name()).setMetadata(Utils.toArray(metadata)));
    }
    return protocolSet;
}

Example 39 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class RangeAssignorTest method testStaticMemberRangeAssignmentPersistent.

@Test
public void testStaticMemberRangeAssignmentPersistent() {
    Map<String, Integer> partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(5, 4);
    Map<String, Subscription> consumers = new HashMap<>();
    for (MemberInfo m : staticMemberInfos) {
        Subscription subscription = new Subscription(topics(topic1, topic2), null, Collections.emptyList());
        subscription.setGroupInstanceId(m.groupInstanceId);
        consumers.put(m.memberId, subscription);
    }
    // Consumer 4 is a dynamic member.
    String consumer4 = "consumer4";
    consumers.put(consumer4, new Subscription(topics(topic1, topic2)));
    Map<String, List<TopicPartition>> expectedAssignment = new HashMap<>();
    // Have 3 static members instance1, instance2, instance3 to be persistent
    // across generations. Their assignment shall be the same.
    expectedAssignment.put(consumer1, partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0)));
    expectedAssignment.put(consumer2, partitions(tp(topic1, 2), tp(topic2, 1)));
    expectedAssignment.put(consumer3, partitions(tp(topic1, 3), tp(topic2, 2)));
    expectedAssignment.put(consumer4, partitions(tp(topic1, 4), tp(topic2, 3)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, consumers);
    assertEquals(expectedAssignment, assignment);
    // Replace dynamic member 4 with a new dynamic member 5.
    consumers.remove(consumer4);
    String consumer5 = "consumer5";
    consumers.put(consumer5, new Subscription(topics(topic1, topic2)));
    expectedAssignment.remove(consumer4);
    expectedAssignment.put(consumer5, partitions(tp(topic1, 4), tp(topic2, 3)));
    assignment = assignor.assign(partitionsPerTopic, consumers);
    assertEquals(expectedAssignment, assignment);
}

Example 40 with Subscription

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Subscription in project kafka by apache.

the class RangeAssignorTest method testOneStaticConsumerAndOneDynamicConsumerTwoTopicsSixPartitions.

@Test
public void testOneStaticConsumerAndOneDynamicConsumerTwoTopicsSixPartitions() {
    // although consumer high has a higher rank than low, consumer low will win the comparison
    // because it has instance id while consumer 2 doesn't.
    String consumerIdLow = "consumer-b";
    String consumerIdHigh = "consumer-a";
    Map<String, Integer> partitionsPerTopic = setupPartitionsPerTopicWithTwoTopics(3, 3);
    Map<String, Subscription> consumers = new HashMap<>();
    Subscription consumerLowSubscription = new Subscription(topics(topic1, topic2), null, Collections.emptyList());
    consumerLowSubscription.setGroupInstanceId(Optional.of(instance1));
    consumers.put(consumerIdLow, consumerLowSubscription);
    consumers.put(consumerIdHigh, new Subscription(topics(topic1, topic2)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, consumers);
    assertAssignment(partitions(tp(topic1, 0), tp(topic1, 1), tp(topic2, 0), tp(topic2, 1)), assignment.get(consumerIdLow));
    assertAssignment(partitions(tp(topic1, 2), tp(topic2, 2)), assignment.get(consumerIdHigh));
}