Examples with TopicsInfo org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo used on opensource projects

Example 6 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class RepartitionTopics method computePartitionCount.

private Integer computePartitionCount(final Map<String, InternalTopicConfig> repartitionTopicMetadata, final Collection<TopicsInfo> topicGroups, final Cluster clusterMetadata, final String repartitionSourceTopic) {
    Integer partitionCount = null;
    // try set the number of partitions for this repartition topic if it is not set yet
    for (final TopicsInfo topicsInfo : topicGroups) {
        final Set<String> sinkTopics = topicsInfo.sinkTopics;
        if (sinkTopics.contains(repartitionSourceTopic)) {
            // use the maximum of all its source topic partitions as the number of partitions
            for (final String upstreamSourceTopic : topicsInfo.sourceTopics) {
                Integer numPartitionsCandidate = null;
                // map().join().join(map())
                if (repartitionTopicMetadata.containsKey(upstreamSourceTopic)) {
                    if (repartitionTopicMetadata.get(upstreamSourceTopic).numberOfPartitions().isPresent()) {
                        numPartitionsCandidate = repartitionTopicMetadata.get(upstreamSourceTopic).numberOfPartitions().get();
                    }
                } else {
                    final Integer count = clusterMetadata.partitionCountForTopic(upstreamSourceTopic);
                    if (count == null) {
                        throw new TaskAssignmentException("No partition count found for source topic " + upstreamSourceTopic + ", but it should have been.");
                    }
                    numPartitionsCandidate = count;
                }
                if (numPartitionsCandidate != null) {
                    if (partitionCount == null || numPartitionsCandidate > partitionCount) {
                        partitionCount = numPartitionsCandidate;
                    }
                }
            }
        }
    }
    return partitionCount;
}

Example 7 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class StreamsPartitionAssignor method assign.

/*
     * This assigns tasks to consumer clients in the following steps.
     *
     * 0. decode the subscriptions to assemble the metadata for each client and check for version probing
     *
     * 1. check all repartition source topics and use internal topic manager to make sure
     *    they have been created with the right number of partitions. Also verify and/or create
     *    any changelog topics with the correct number of partitions.
     *
     * 2. use the partition grouper to generate tasks along with their assigned partitions, then use
     *    the configured TaskAssignor to construct the mapping of tasks to clients.
     *
     * 3. construct the global mapping of host to partitions to enable query routing.
     *
     * 4. within each client, assign tasks to consumer clients.
     */
@Override
public GroupAssignment assign(final Cluster metadata, final GroupSubscription groupSubscription) {
    final Map<String, Subscription> subscriptions = groupSubscription.groupSubscription();
    // ---------------- Step Zero ---------------- //
    // construct the client metadata from the decoded subscription info
    final Map<UUID, ClientMetadata> clientMetadataMap = new HashMap<>();
    final Set<TopicPartition> allOwnedPartitions = new HashSet<>();
    int minReceivedMetadataVersion = LATEST_SUPPORTED_VERSION;
    int minSupportedMetadataVersion = LATEST_SUPPORTED_VERSION;
    boolean shutdownRequested = false;
    boolean assignmentErrorFound = false;
    int futureMetadataVersion = UNKNOWN;
    for (final Map.Entry<String, Subscription> entry : subscriptions.entrySet()) {
        final String consumerId = entry.getKey();
        final Subscription subscription = entry.getValue();
        final SubscriptionInfo info = SubscriptionInfo.decode(subscription.userData());
        final int usedVersion = info.version();
        if (info.errorCode() == AssignorError.SHUTDOWN_REQUESTED.code()) {
            shutdownRequested = true;
        }
        minReceivedMetadataVersion = updateMinReceivedVersion(usedVersion, minReceivedMetadataVersion);
        minSupportedMetadataVersion = updateMinSupportedVersion(info.latestSupportedVersion(), minSupportedMetadataVersion);
        final UUID processId;
        if (usedVersion > LATEST_SUPPORTED_VERSION) {
            futureMetadataVersion = usedVersion;
            processId = FUTURE_ID;
            if (!clientMetadataMap.containsKey(FUTURE_ID)) {
                clientMetadataMap.put(FUTURE_ID, new ClientMetadata(null));
            }
        } else {
            processId = info.processId();
        }
        ClientMetadata clientMetadata = clientMetadataMap.get(processId);
        // create the new client metadata if necessary
        if (clientMetadata == null) {
            clientMetadata = new ClientMetadata(info.userEndPoint());
            clientMetadataMap.put(info.processId(), clientMetadata);
        }
        // add the consumer and any info in its subscription to the client
        clientMetadata.addConsumer(consumerId, subscription.ownedPartitions());
        final int prevSize = allOwnedPartitions.size();
        allOwnedPartitions.addAll(subscription.ownedPartitions());
        if (allOwnedPartitions.size() < prevSize + subscription.ownedPartitions().size()) {
            assignmentErrorFound = true;
        }
        clientMetadata.addPreviousTasksAndOffsetSums(consumerId, info.taskOffsetSums());
    }
    if (assignmentErrorFound) {
        log.warn("The previous assignment contains a partition more than once. " + "\t Mapping: {}", subscriptions);
    }
    try {
        final boolean versionProbing = checkMetadataVersions(minReceivedMetadataVersion, minSupportedMetadataVersion, futureMetadataVersion);
        log.debug("Constructed client metadata {} from the member subscriptions.", clientMetadataMap);
        if (shutdownRequested) {
            return new GroupAssignment(errorAssignment(clientMetadataMap, AssignorError.SHUTDOWN_REQUESTED.code()));
        }
        // parse the topology to determine the repartition source topics,
        // making sure they are created with the number of partitions as
        // the maximum of the depending sub-topologies source topics' number of partitions
        final RepartitionTopics repartitionTopics = prepareRepartitionTopics(metadata);
        final Map<TopicPartition, PartitionInfo> allRepartitionTopicPartitions = repartitionTopics.topicPartitionsInfo();
        final Cluster fullMetadata = metadata.withPartitions(allRepartitionTopicPartitions);
        log.debug("Created repartition topics {} from the parsed topology.", allRepartitionTopicPartitions.values());
        // ---------------- Step Two ---------------- //
        // construct the assignment of tasks to clients
        final Map<Subtopology, TopicsInfo> topicGroups = taskManager.topologyMetadata().subtopologyTopicsInfoMapExcluding(repartitionTopics.topologiesWithMissingInputTopics());
        final Set<String> allSourceTopics = new HashSet<>();
        final Map<Subtopology, Set<String>> sourceTopicsByGroup = new HashMap<>();
        for (final Map.Entry<Subtopology, TopicsInfo> entry : topicGroups.entrySet()) {
            allSourceTopics.addAll(entry.getValue().sourceTopics);
            sourceTopicsByGroup.put(entry.getKey(), entry.getValue().sourceTopics);
        }
        // get the tasks as partition groups from the partition grouper
        final Map<TaskId, Set<TopicPartition>> partitionsForTask = partitionGrouper.partitionGroups(sourceTopicsByGroup, fullMetadata);
        final Set<TaskId> statefulTasks = new HashSet<>();
        final boolean probingRebalanceNeeded = assignTasksToClients(fullMetadata, allSourceTopics, topicGroups, clientMetadataMap, partitionsForTask, statefulTasks);
        // ---------------- Step Three ---------------- //
        // construct the global partition assignment per host map
        final Map<HostInfo, Set<TopicPartition>> partitionsByHost = new HashMap<>();
        final Map<HostInfo, Set<TopicPartition>> standbyPartitionsByHost = new HashMap<>();
        if (minReceivedMetadataVersion >= 2) {
            populatePartitionsByHostMaps(partitionsByHost, standbyPartitionsByHost, partitionsForTask, clientMetadataMap);
        }
        streamsMetadataState.onChange(partitionsByHost, standbyPartitionsByHost, fullMetadata);
        // ---------------- Step Four ---------------- //
        // compute the assignment of tasks to threads within each client and build the final group assignment
        final Map<String, Assignment> assignment = computeNewAssignment(statefulTasks, clientMetadataMap, partitionsForTask, partitionsByHost, standbyPartitionsByHost, allOwnedPartitions, minReceivedMetadataVersion, minSupportedMetadataVersion, versionProbing, probingRebalanceNeeded);
        return new GroupAssignment(assignment);
    } catch (final MissingSourceTopicException e) {
        log.error("Caught an error in the task assignment. Returning an error assignment.", e);
        return new GroupAssignment(errorAssignment(clientMetadataMap, AssignorError.INCOMPLETE_SOURCE_TOPIC_METADATA.code()));
    } catch (final TaskAssignmentException e) {
        log.error("Caught an error in the task assignment. Returning an error assignment.", e);
        return new GroupAssignment(errorAssignment(clientMetadataMap, AssignorError.ASSIGNMENT_ERROR.code()));
    }
}

Example 8 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class ChangelogTopics method setup.

public void setup() {
    // add tasks to state change log topic subscribers
    final Map<String, InternalTopicConfig> changelogTopicMetadata = new HashMap<>();
    for (final Map.Entry<Subtopology, TopicsInfo> entry : topicGroups.entrySet()) {
        final Subtopology subtopology = entry.getKey();
        final TopicsInfo topicsInfo = entry.getValue();
        final Set<TaskId> topicGroupTasks = tasksForTopicGroup.get(subtopology);
        if (topicGroupTasks == null) {
            log.debug("No tasks found for subtopology {}", subtopology);
            continue;
        } else if (topicsInfo.stateChangelogTopics.isEmpty()) {
            continue;
        }
        for (final TaskId task : topicGroupTasks) {
            final Set<TopicPartition> changelogTopicPartitions = topicsInfo.stateChangelogTopics.keySet().stream().map(topic -> new TopicPartition(topic, task.partition())).collect(Collectors.toSet());
            changelogPartitionsForStatefulTask.put(task, changelogTopicPartitions);
        }
        for (final InternalTopicConfig topicConfig : topicsInfo.nonSourceChangelogTopics()) {
            // the expected number of partitions is the max value of TaskId.partition + 1
            int numPartitions = UNKNOWN;
            for (final TaskId task : topicGroupTasks) {
                if (numPartitions < task.partition() + 1) {
                    numPartitions = task.partition() + 1;
                }
            }
            topicConfig.setNumberOfPartitions(numPartitions);
            changelogTopicMetadata.put(topicConfig.name(), topicConfig);
        }
        sourceTopicBasedChangelogTopics.addAll(topicsInfo.sourceTopicChangelogs());
    }
    final Set<String> newlyCreatedChangelogTopics = internalTopicManager.makeReady(changelogTopicMetadata);
    log.debug("Created state changelog topics {} from the parsed topology.", changelogTopicMetadata.values());
    for (final Map.Entry<TaskId, Set<TopicPartition>> entry : changelogPartitionsForStatefulTask.entrySet()) {
        final TaskId taskId = entry.getKey();
        final Set<TopicPartition> topicPartitions = entry.getValue();
        for (final TopicPartition topicPartition : topicPartitions) {
            if (!newlyCreatedChangelogTopics.contains(topicPartition.topic())) {
                preExistingChangelogPartitionsForTask.computeIfAbsent(taskId, task -> new HashSet<>()).add(topicPartition);
                if (!sourceTopicBasedChangelogTopics.contains(topicPartition.topic())) {
                    preExistingNonSourceTopicBasedChangelogPartitions.add(topicPartition);
                } else {
                    preExistingSourceTopicBasedChangelogPartitions.add(topicPartition);
                }
            }
        }
    }
}

Example 9 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class RepartitionTopics method setRepartitionSourceTopicPartitionCount.

/**
 * Computes the number of partitions and sets it for each repartition topic in repartitionTopicMetadata
 */
private void setRepartitionSourceTopicPartitionCount(final Map<String, InternalTopicConfig> repartitionTopicMetadata, final Collection<TopicsInfo> topicGroups, final Cluster clusterMetadata) {
    boolean partitionCountNeeded;
    do {
        partitionCountNeeded = false;
        // avoid infinitely looping without making any progress on unknown repartitions
        boolean progressMadeThisIteration = false;
        for (final TopicsInfo topicsInfo : topicGroups) {
            for (final String repartitionSourceTopic : topicsInfo.repartitionSourceTopics.keySet()) {
                final Optional<Integer> repartitionSourceTopicPartitionCount = repartitionTopicMetadata.get(repartitionSourceTopic).numberOfPartitions();
                if (!repartitionSourceTopicPartitionCount.isPresent()) {
                    final Integer numPartitions = computePartitionCount(repartitionTopicMetadata, topicGroups, clusterMetadata, repartitionSourceTopic);
                    if (numPartitions == null) {
                        partitionCountNeeded = true;
                        log.trace("Unable to determine number of partitions for {}, another iteration is needed", repartitionSourceTopic);
                    } else {
                        log.trace("Determined number of partitions for {} to be {}", repartitionSourceTopic, numPartitions);
                        repartitionTopicMetadata.get(repartitionSourceTopic).setNumberOfPartitions(numPartitions);
                        progressMadeThisIteration = true;
                    }
                }
            }
        }
        if (!progressMadeThisIteration && partitionCountNeeded) {
            log.error("Unable to determine the number of partitions of all repartition topics, most likely a source topic is missing or pattern doesn't match any topics\n" + "topic groups: {}\n" + "cluster topics: {}.", topicGroups, clusterMetadata.topics());
            throw new TaskAssignmentException("Failed to compute number of partitions for all repartition topics, " + "make sure all user input topics are created and all Pattern subscriptions match at least one topic in the cluster");
        }
    } while (partitionCountNeeded);
}

Example 10 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class ChangelogTopicsTest method shouldOnlyContainPreExistingNonSourceBasedChangelogs.

@Test
public void shouldOnlyContainPreExistingNonSourceBasedChangelogs() {
    expect(internalTopicManager.makeReady(mkMap(mkEntry(CHANGELOG_TOPIC_NAME1, CHANGELOG_TOPIC_CONFIG)))).andStubReturn(Collections.emptySet());
    final Map<Subtopology, TopicsInfo> topicGroups = mkMap(mkEntry(SUBTOPOLOGY_0, TOPICS_INFO1));
    final Set<TaskId> tasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2);
    final Map<Subtopology, Set<TaskId>> tasksForTopicGroup = mkMap(mkEntry(SUBTOPOLOGY_0, tasks));
    replay(internalTopicManager);
    final ChangelogTopics changelogTopics = new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, "[test] ");
    changelogTopics.setup();
    verify(internalTopicManager);
    assertThat(CHANGELOG_TOPIC_CONFIG.numberOfPartitions().orElse(Integer.MIN_VALUE), is(3));
    final TopicPartition changelogPartition0 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 0);
    final TopicPartition changelogPartition1 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 1);
    final TopicPartition changelogPartition2 = new TopicPartition(CHANGELOG_TOPIC_NAME1, 2);
    assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_0), is(mkSet(changelogPartition0)));
    assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_1), is(mkSet(changelogPartition1)));
    assertThat(changelogTopics.preExistingPartitionsFor(TASK_0_2), is(mkSet(changelogPartition2)));
    assertThat(changelogTopics.preExistingSourceTopicBasedPartitions(), is(Collections.emptySet()));
    assertThat(changelogTopics.preExistingNonSourceTopicBasedPartitions(), is(mkSet(changelogPartition0, changelogPartition1, changelogPartition2)));
}