Examples with TopicPartition org.apache.kafka.common.TopicPartition used on opensource projects

Example 1 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class ConsumerCoordinator method doAutoCommitOffsetsAsync.

private void doAutoCommitOffsetsAsync() {
    Map<TopicPartition, OffsetAndMetadata> allConsumedOffsets = subscriptions.allConsumed();
    log.debug("Sending asynchronous auto-commit of offsets {} for group {}", allConsumedOffsets, groupId);
    commitOffsetsAsync(allConsumedOffsets, new OffsetCommitCallback() {

        @Override
        public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
            if (exception != null) {
                log.warn("Auto-commit of offsets {} failed for group {}: {}", offsets, groupId, exception.getMessage());
                if (exception instanceof RetriableException)
                    nextAutoCommitDeadline = Math.min(time.milliseconds() + retryBackoffMs, nextAutoCommitDeadline);
            } else {
                log.debug("Completed auto-commit of offsets {} for group {}", offsets, groupId);
            }
        }
    });
}

Example 2 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class ConsumerCoordinator method performAssignment.

@Override
protected Map<String, ByteBuffer> performAssignment(String leaderId, String assignmentStrategy, Map<String, ByteBuffer> allSubscriptions) {
    PartitionAssignor assignor = lookupAssignor(assignmentStrategy);
    if (assignor == null)
        throw new IllegalStateException("Coordinator selected invalid assignment protocol: " + assignmentStrategy);
    Set<String> allSubscribedTopics = new HashSet<>();
    Map<String, Subscription> subscriptions = new HashMap<>();
    for (Map.Entry<String, ByteBuffer> subscriptionEntry : allSubscriptions.entrySet()) {
        Subscription subscription = ConsumerProtocol.deserializeSubscription(subscriptionEntry.getValue());
        subscriptions.put(subscriptionEntry.getKey(), subscription);
        allSubscribedTopics.addAll(subscription.topics());
    }
    // the leader will begin watching for changes to any of the topics the group is interested in,
    // which ensures that all metadata changes will eventually be seen
    this.subscriptions.groupSubscribe(allSubscribedTopics);
    metadata.setTopics(this.subscriptions.groupSubscription());
    // update metadata (if needed) and keep track of the metadata used for assignment so that
    // we can check after rebalance completion whether anything has changed
    client.ensureFreshMetadata();
    isLeader = true;
    log.debug("Performing assignment for group {} using strategy {} with subscriptions {}", groupId, assignor.name(), subscriptions);
    Map<String, Assignment> assignment = assignor.assign(metadata.fetch(), subscriptions);
    // user-customized assignor may have created some topics that are not in the subscription list
    // and assign their partitions to the members; in this case we would like to update the leader's
    // own metadata with the newly added topics so that it will not trigger a subsequent rebalance
    // when these topics gets updated from metadata refresh.
    //
    // TODO: this is a hack and not something we want to support long-term unless we push regex into the protocol
    //       we may need to modify the PartitionAssingor API to better support this case.
    Set<String> assignedTopics = new HashSet<>();
    for (Assignment assigned : assignment.values()) {
        for (TopicPartition tp : assigned.partitions()) assignedTopics.add(tp.topic());
    }
    if (!assignedTopics.containsAll(allSubscribedTopics)) {
        Set<String> notAssignedTopics = new HashSet<>(allSubscribedTopics);
        notAssignedTopics.removeAll(assignedTopics);
        log.warn("The following subscribed topics are not assigned to any members in the group {} : {} ", groupId, notAssignedTopics);
    }
    if (!allSubscribedTopics.containsAll(assignedTopics)) {
        Set<String> newlyAddedTopics = new HashSet<>(assignedTopics);
        newlyAddedTopics.removeAll(allSubscribedTopics);
        log.info("The following not-subscribed topics are assigned to group {}, and their metadata will be " + "fetched from the brokers : {}", groupId, newlyAddedTopics);
        allSubscribedTopics.addAll(assignedTopics);
        this.subscriptions.groupSubscribe(allSubscribedTopics);
        metadata.setTopics(this.subscriptions.groupSubscription());
        client.ensureFreshMetadata();
    }
    assignmentSnapshot = metadataSnapshot;
    log.debug("Finished assignment for group {}: {}", groupId, assignment);
    Map<String, ByteBuffer> groupAssignment = new HashMap<>();
    for (Map.Entry<String, Assignment> assignmentEntry : assignment.entrySet()) {
        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(assignmentEntry.getValue());
        groupAssignment.put(assignmentEntry.getKey(), buffer);
    }
    return groupAssignment;
}

Example 3 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class ConsumerProtocol method asMap.

private static Map<String, List<Integer>> asMap(Collection<TopicPartition> partitions) {
    Map<String, List<Integer>> partitionMap = new HashMap<>();
    for (TopicPartition partition : partitions) {
        String topic = partition.topic();
        List<Integer> topicPartitions = partitionMap.get(topic);
        if (topicPartitions == null) {
            topicPartitions = new ArrayList<>();
            partitionMap.put(topic, topicPartitions);
        }
        topicPartitions.add(partition.partition());
    }
    return partitionMap;
}

Example 4 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class ConsumerProtocol method deserializeAssignment.

public static PartitionAssignor.Assignment deserializeAssignment(ByteBuffer buffer) {
    Struct header = CONSUMER_PROTOCOL_HEADER_SCHEMA.read(buffer);
    Short version = header.getShort(VERSION_KEY_NAME);
    checkVersionCompatibility(version);
    Struct struct = ASSIGNMENT_V0.read(buffer);
    ByteBuffer userData = struct.getBytes(USER_DATA_KEY_NAME);
    List<TopicPartition> partitions = new ArrayList<>();
    for (Object structObj : struct.getArray(TOPIC_PARTITIONS_KEY_NAME)) {
        Struct assignment = (Struct) structObj;
        String topic = assignment.getString(TOPIC_KEY_NAME);
        for (Object partitionObj : assignment.getArray(PARTITIONS_KEY_NAME)) {
            Integer partition = (Integer) partitionObj;
            partitions.add(new TopicPartition(topic, partition));
        }
    }
    return new PartitionAssignor.Assignment(partitions, userData);
}

Example 5 with TopicPartition

use of org.apache.kafka.common.TopicPartition in project kafka by apache.

the class Fetcher method beginningOrEndOffset.

private Map<TopicPartition, Long> beginningOrEndOffset(Collection<TopicPartition> partitions, long timestamp, long timeout) {
    Map<TopicPartition, Long> timestampsToSearch = new HashMap<>();
    for (TopicPartition tp : partitions) timestampsToSearch.put(tp, timestamp);
    Map<TopicPartition, Long> result = new HashMap<>();
    for (Map.Entry<TopicPartition, OffsetData> entry : retrieveOffsetsByTimes(timestampsToSearch, timeout, false).entrySet()) {
        result.put(entry.getKey(), entry.getValue().offset);
    }
    return result;
}