Examples with Subscription org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription used on opensource projects

Example 16 with Subscription

use of org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerProtocolTest method deserializeNewSubscriptionVersion.

@Test
public void deserializeNewSubscriptionVersion() {
    // verify that a new version which adds a field is still parseable
    short version = 100;
    Schema subscriptionSchemaV100 = new Schema(new Field(ConsumerProtocol.TOPICS_KEY_NAME, new ArrayOf(Type.STRING)), new Field(ConsumerProtocol.USER_DATA_KEY_NAME, Type.BYTES), new Field("foo", Type.STRING));
    Struct subscriptionV100 = new Struct(subscriptionSchemaV100);
    subscriptionV100.set(ConsumerProtocol.TOPICS_KEY_NAME, new Object[] { "topic" });
    subscriptionV100.set(ConsumerProtocol.USER_DATA_KEY_NAME, ByteBuffer.wrap(new byte[0]));
    subscriptionV100.set("foo", "bar");
    Struct headerV100 = new Struct(ConsumerProtocol.CONSUMER_PROTOCOL_HEADER_SCHEMA);
    headerV100.set(ConsumerProtocol.VERSION_KEY_NAME, version);
    ByteBuffer buffer = ByteBuffer.allocate(subscriptionV100.sizeOf() + headerV100.sizeOf());
    headerV100.writeTo(buffer);
    subscriptionV100.writeTo(buffer);
    buffer.flip();
    Subscription subscription = ConsumerProtocol.deserializeSubscription(buffer);
    assertEquals(Arrays.asList("topic"), subscription.topics());
}

Example 17 with Subscription

use of org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinator method metadata.

@Override
public List<ProtocolMetadata> metadata() {
    this.joinedSubscription = subscriptions.subscription();
    List<ProtocolMetadata> metadataList = new ArrayList<>();
    for (PartitionAssignor assignor : assignors) {
        Subscription subscription = assignor.subscription(joinedSubscription);
        ByteBuffer metadata = ConsumerProtocol.serializeSubscription(subscription);
        metadataList.add(new ProtocolMetadata(assignor.name(), metadata));
    }
    return metadataList;
}

Example 18 with Subscription

use of org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinator method performAssignment.

@Override
protected Map<String, ByteBuffer> performAssignment(String leaderId, String assignmentStrategy, Map<String, ByteBuffer> allSubscriptions) {
    PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
    if (assignor == null)
        throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
    Set<String> allSubscribedTopics = new HashSet<>();
    Map<String, Subscription> subscriptions = new HashMap<>();
    for (Map.Entry<String, ByteBuffer> subscriptionEntry : allSubscriptions.entrySet()) {
        Subscription subscription = ConsumerProtocol.deserializeSubscription(subscriptionEntry.getValue());
        subscriptions.put(subscriptionEntry.getKey(), subscription);
        allSubscribedTopics.addAll(subscription.topics());
    }
    // the leader will begin watching for changes to any of the topics the group is interested in,
    // which ensures that all metadata changes will eventually be seen
    this.subscriptions.groupSubscribe(allSubscribedTopics);
    metadata.setTopics(this.subscriptions.groupSubscription());
    // update metadata (if needed) and keep track of the metadata used for assignment so that
    // we can check after rebalance completion whether anything has changed
    client.ensureFreshMetadata();
    isLeader = true;
    log.debug("Performing assignment using strategy {} with subscriptions {}", assignor.name(), subscriptions);
    Map<String, Assignment> assignment = assignor.assign(metadata.fetch(), subscriptions);
    // user-customized assignor may have created some topics that are not in the subscription list
    // and assign their partitions to the members; in this case we would like to update the leader's
    // own metadata with the newly added topics so that it will not trigger a subsequent rebalance
    // when these topics gets updated from metadata refresh.
    // 
    // TODO: this is a hack and not something we want to support long-term unless we push regex into the protocol
    // we may need to modify the PartitionAssignor API to better support this case.
    Set<String> assignedTopics = new HashSet<>();
    for (Assignment assigned : assignment.values()) {
        for (TopicPartition tp : assigned.partitions()) assignedTopics.add(tp.topic());
    }
    if (!assignedTopics.containsAll(allSubscribedTopics)) {
        Set<String> notAssignedTopics = new HashSet<>(allSubscribedTopics);
        notAssignedTopics.removeAll(assignedTopics);
        log.warn("The following subscribed topics are not assigned to any members: {} ", notAssignedTopics);
    }
    if (!allSubscribedTopics.containsAll(assignedTopics)) {
        Set<String> newlyAddedTopics = new HashSet<>(assignedTopics);
        newlyAddedTopics.removeAll(allSubscribedTopics);
        log.info("The following not-subscribed topics are assigned, and their metadata will be " + "fetched from the brokers: {}", newlyAddedTopics);
        allSubscribedTopics.addAll(assignedTopics);
        this.subscriptions.groupSubscribe(allSubscribedTopics);
        metadata.setTopics(this.subscriptions.groupSubscription());
        client.ensureFreshMetadata();
    }
    assignmentSnapshot = metadataSnapshot;
    log.debug("Finished assignment for group: {}", assignment);
    Map<String, ByteBuffer> groupAssignment = new HashMap<>();
    for (Map.Entry<String, Assignment> assignmentEntry : assignment.entrySet()) {
        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(assignmentEntry.getValue());
        groupAssignment.put(assignmentEntry.getKey(), buffer);
    }
    return groupAssignment;
}

Example 19 with Subscription

use of org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription in project apache-kafka-on-k8s by banzaicloud.

the class RangeAssignorTest method testTwoConsumersOneTopicTwoPartitions.

@Test
public void testTwoConsumersOneTopicTwoPartitions() {
    String topic = "topic";
    String consumer1 = "consumer1";
    String consumer2 = "consumer2";
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic, 2);
    Map<String, Subscription> consumers = new HashMap<>();
    consumers.put(consumer1, new Subscription(topics(topic)));
    consumers.put(consumer2, new Subscription(topics(topic)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, consumers);
    assertAssignment(partitions(tp(topic, 0)), assignment.get(consumer1));
    assertAssignment(partitions(tp(topic, 1)), assignment.get(consumer2));
}

Example 20 with Subscription

use of org.apache.kafka.clients.consumer.internals.PartitionAssignor.Subscription in project apache-kafka-on-k8s by banzaicloud.

the class RangeAssignorTest method testTwoConsumersOneTopicOnePartition.

@Test
public void testTwoConsumersOneTopicOnePartition() {
    String topic = "topic";
    String consumer1 = "consumer1";
    String consumer2 = "consumer2";
    Map<String, Integer> partitionsPerTopic = new HashMap<>();
    partitionsPerTopic.put(topic, 1);
    Map<String, Subscription> consumers = new HashMap<>();
    consumers.put(consumer1, new Subscription(topics(topic)));
    consumers.put(consumer2, new Subscription(topics(topic)));
    Map<String, List<TopicPartition>> assignment = assignor.assign(partitionsPerTopic, consumers);
    assertAssignment(partitions(tp(topic, 0)), assignment.get(consumer1));
    assertAssignment(Collections.<TopicPartition>emptyList(), assignment.get(consumer2));
}