Examples with Assignment org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment used on opensource projects

Example 1 with Assignment

use of org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment in project kafka by apache.

the class ConsumerCoordinator method performAssignment.

@Override
protected Map<String, ByteBuffer> performAssignment(String leaderId, String assignmentStrategy, Map<String, ByteBuffer> allSubscriptions) {
    PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
    if (assignor == null)
        throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
    Set<String> allSubscribedTopics = new HashSet<>();
    Map<String, Subscription> subscriptions = new HashMap<>();
    for (Map.Entry<String, ByteBuffer> subscriptionEntry : allSubscriptions.entrySet()) {
        Subscription subscription = ConsumerProtocol.deserializeSubscription(subscriptionEntry.getValue());
        subscriptions.put(subscriptionEntry.getKey(), subscription);
        allSubscribedTopics.addAll(subscription.topics());
    }
    // the leader will begin watching for changes to any of the topics the group is interested in,
    // which ensures that all metadata changes will eventually be seen
    this.subscriptions.groupSubscribe(allSubscribedTopics);
    metadata.setTopics(this.subscriptions.groupSubscription());
    // update metadata (if needed) and keep track of the metadata used for assignment so that
    // we can check after rebalance completion whether anything has changed
    client.ensureFreshMetadata();
    isLeader = true;
    log.debug("Performing assignment for group {} using strategy {} with subscriptions {}", groupId, assignor.name(), subscriptions);
    Map<String, Assignment> assignment = assignor.assign(metadata.fetch(), subscriptions);
    // user-customized assignor may have created some topics that are not in the subscription list
    // and assign their partitions to the members; in this case we would like to update the leader's
    // own metadata with the newly added topics so that it will not trigger a subsequent rebalance
    // when these topics gets updated from metadata refresh.
    //
    // TODO: this is a hack and not something we want to support long-term unless we push regex into the protocol
    //       we may need to modify the PartitionAssingor API to better support this case.
    Set<String> assignedTopics = new HashSet<>();
    for (Assignment assigned : assignment.values()) {
        for (TopicPartition tp : assigned.partitions()) assignedTopics.add(tp.topic());
    }
    if (!assignedTopics.containsAll(allSubscribedTopics)) {
        Set<String> notAssignedTopics = new HashSet<>(allSubscribedTopics);
        notAssignedTopics.removeAll(assignedTopics);
        log.warn("The following subscribed topics are not assigned to any members in the group {} : {} ", groupId, notAssignedTopics);
    }
    if (!allSubscribedTopics.containsAll(assignedTopics)) {
        Set<String> newlyAddedTopics = new HashSet<>(assignedTopics);
        newlyAddedTopics.removeAll(allSubscribedTopics);
        log.info("The following not-subscribed topics are assigned to group {}, and their metadata will be " + "fetched from the brokers : {}", groupId, newlyAddedTopics);
        allSubscribedTopics.addAll(assignedTopics);
        this.subscriptions.groupSubscribe(allSubscribedTopics);
        metadata.setTopics(this.subscriptions.groupSubscription());
        client.ensureFreshMetadata();
    }
    assignmentSnapshot = metadataSnapshot;
    log.debug("Finished assignment for group {}: {}", groupId, assignment);
    Map<String, ByteBuffer> groupAssignment = new HashMap<>();
    for (Map.Entry<String, Assignment> assignmentEntry : assignment.entrySet()) {
        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(assignmentEntry.getValue());
        groupAssignment.put(assignmentEntry.getKey(), buffer);
    }
    return groupAssignment;
}

Example 2 with Assignment

use of org.apache.kafka.clients.consumer.internals.PartitionAssignor.Assignment in project kafka by apache.

the class ConsumerCoordinator method onJoinComplete.

@Override
protected void onJoinComplete(int generation, String memberId, String assignmentStrategy, ByteBuffer assignmentBuffer) {
    // only the leader is responsible for monitoring for metadata changes (i.e. partition changes)
    if (!isLeader)
        assignmentSnapshot = null;
    PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
    if (assignor == null)
        throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
    Assignment assignment = ConsumerProtocol.deserializeAssignment(assignmentBuffer);
    // set the flag to refresh last committed offsets
    subscriptions.needRefreshCommits();
    // update partition assignment
    subscriptions.assignFromSubscribed(assignment.partitions());
    // check if the assignment contains some topics that were not in the original
    // subscription, if yes we will obey what leader has decided and add these topics
    // into the subscriptions as long as they still match the subscribed pattern
    //
    // TODO this part of the logic should be removed once we allow regex on leader assign
    Set<String> addedTopics = new HashSet<>();
    for (TopicPartition tp : subscriptions.assignedPartitions()) {
        if (!joinedSubscription.contains(tp.topic()))
            addedTopics.add(tp.topic());
    }
    if (!addedTopics.isEmpty()) {
        Set<String> newSubscription = new HashSet<>(subscriptions.subscription());
        Set<String> newJoinedSubscription = new HashSet<>(joinedSubscription);
        newSubscription.addAll(addedTopics);
        newJoinedSubscription.addAll(addedTopics);
        this.subscriptions.subscribeFromPattern(newSubscription);
        this.joinedSubscription = newJoinedSubscription;
    }
    // update the metadata and enforce a refresh to make sure the fetcher can start
    // fetching data in the next iteration
    this.metadata.setTopics(subscriptions.groupSubscription());
    client.ensureFreshMetadata();
    // give the assignor a chance to update internal state based on the received assignment
    assignor.onAssignment(assignment);
    // reschedule the auto commit starting from now
    this.nextAutoCommitDeadline = time.milliseconds() + autoCommitIntervalMs;
    // execute the user's callback after rebalance
    ConsumerRebalanceListener listener = subscriptions.listener();
    log.info("Setting newly assigned partitions {} for group {}", subscriptions.assignedPartitions(), groupId);
    try {
        Set<TopicPartition> assigned = new HashSet<>(subscriptions.assignedPartitions());
        listener.onPartitionsAssigned(assigned);
    } catch (WakeupException | InterruptException e) {
        throw e;
    } catch (Exception e) {
        log.error("User provided listener {} for group {} failed on partition assignment", listener.getClass().getName(), groupId, e);
    }
}