Examples with Subtopology org.apache.kafka.streams.processor.internals.TopologyMetadata.Subtopology used on opensource projects

Example 1 with Subtopology

use of org.apache.kafka.streams.processor.internals.TopologyMetadata.Subtopology in project kafka by apache.

the class StreamsPartitionAssignorTest method testAssignWithStates.

@Test
public void testAssignWithStates() {
    builder.addSource(null, "source1", null, null, null, "topic1");
    builder.addSource(null, "source2", null, null, null, "topic2");
    builder.addProcessor("processor-1", new MockApiProcessorSupplier<>(), "source1");
    builder.addStateStore(new MockKeyValueStoreBuilder("store1", false), "processor-1");
    builder.addProcessor("processor-2", new MockApiProcessorSupplier<>(), "source2");
    builder.addStateStore(new MockKeyValueStoreBuilder("store2", false), "processor-2");
    builder.addStateStore(new MockKeyValueStoreBuilder("store3", false), "processor-2");
    final List<String> topics = asList("topic1", "topic2");
    final List<TaskId> tasks = asList(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2);
    adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(asList(APPLICATION_ID + "-store1-changelog", APPLICATION_ID + "-store2-changelog", APPLICATION_ID + "-store3-changelog"), asList(3, 3, 3)));
    configureDefault();
    subscriptions.put("consumer10", new Subscription(topics, defaultSubscriptionInfo.encode()));
    subscriptions.put("consumer11", new Subscription(topics, defaultSubscriptionInfo.encode()));
    subscriptions.put("consumer20", new Subscription(topics, getInfo(UUID_2, EMPTY_TASKS, EMPTY_TASKS).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    // check assigned partition size: since there is no previous task and there are two sub-topologies the assignment is random so we cannot check exact match
    assertEquals(2, assignments.get("consumer10").partitions().size());
    assertEquals(2, assignments.get("consumer11").partitions().size());
    assertEquals(2, assignments.get("consumer20").partitions().size());
    final AssignmentInfo info10 = AssignmentInfo.decode(assignments.get("consumer10").userData());
    final AssignmentInfo info11 = AssignmentInfo.decode(assignments.get("consumer11").userData());
    final AssignmentInfo info20 = AssignmentInfo.decode(assignments.get("consumer20").userData());
    assertEquals(2, info10.activeTasks().size());
    assertEquals(2, info11.activeTasks().size());
    assertEquals(2, info20.activeTasks().size());
    final Set<TaskId> allTasks = new HashSet<>();
    allTasks.addAll(info10.activeTasks());
    allTasks.addAll(info11.activeTasks());
    allTasks.addAll(info20.activeTasks());
    assertEquals(new HashSet<>(tasks), allTasks);
    // check tasks for state topics
    final Map<Subtopology, InternalTopologyBuilder.TopicsInfo> topicGroups = builder.subtopologyToTopicsInfo();
    assertEquals(mkSet(TASK_0_0, TASK_0_1, TASK_0_2), tasksForState("store1", tasks, topicGroups));
    assertEquals(mkSet(TASK_1_0, TASK_1_1, TASK_1_2), tasksForState("store2", tasks, topicGroups));
    assertEquals(mkSet(TASK_1_0, TASK_1_1, TASK_1_2), tasksForState("store3", tasks, topicGroups));
}

Example 2 with Subtopology

use of org.apache.kafka.streams.processor.internals.TopologyMetadata.Subtopology in project kafka by apache.

the class PartitionGrouperTest method shouldComputeGroupingForSingleGroupWithMultipleTopics.

@Test
public void shouldComputeGroupingForSingleGroupWithMultipleTopics() {
    final PartitionGrouper grouper = new PartitionGrouper();
    final Map<TaskId, Set<TopicPartition>> expectedPartitionsForTask = new HashMap<>();
    final Map<Subtopology, Set<String>> topicGroups = new HashMap<>();
    topicGroups.put(SUBTOPOLOGY_0, mkSet("topic1", "topic2"));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_0.nodeGroupId, 0, SUBTOPOLOGY_0.namedTopology), mkSet(new TopicPartition("topic1", 0), new TopicPartition("topic2", 0)));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_0.nodeGroupId, 1, SUBTOPOLOGY_0.namedTopology), mkSet(new TopicPartition("topic1", 1), new TopicPartition("topic2", 1)));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_0.nodeGroupId, 2, SUBTOPOLOGY_0.namedTopology), mkSet(new TopicPartition("topic1", 2)));
    assertEquals(expectedPartitionsForTask, grouper.partitionGroups(topicGroups, metadata));
}

Example 3 with Subtopology

use of org.apache.kafka.streams.processor.internals.TopologyMetadata.Subtopology in project kafka by apache.

the class PartitionGrouperTest method shouldComputeGroupingForTwoGroups.

@Test
public void shouldComputeGroupingForTwoGroups() {
    final PartitionGrouper grouper = new PartitionGrouper();
    final Map<TaskId, Set<TopicPartition>> expectedPartitionsForTask = new HashMap<>();
    final Map<Subtopology, Set<String>> topicGroups = new HashMap<>();
    topicGroups.put(SUBTOPOLOGY_0, mkSet("topic1"));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_0.nodeGroupId, 0, SUBTOPOLOGY_0.namedTopology), mkSet(new TopicPartition("topic1", 0)));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_0.nodeGroupId, 1, SUBTOPOLOGY_0.namedTopology), mkSet(new TopicPartition("topic1", 1)));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_0.nodeGroupId, 2, SUBTOPOLOGY_0.namedTopology), mkSet(new TopicPartition("topic1", 2)));
    topicGroups.put(SUBTOPOLOGY_1, mkSet("topic2"));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_1.nodeGroupId, 0, SUBTOPOLOGY_1.namedTopology), mkSet(new TopicPartition("topic2", 0)));
    expectedPartitionsForTask.put(new TaskId(SUBTOPOLOGY_1.nodeGroupId, 1, SUBTOPOLOGY_1.namedTopology), mkSet(new TopicPartition("topic2", 1)));
    assertEquals(expectedPartitionsForTask, grouper.partitionGroups(topicGroups, metadata));
}

Example 4 with Subtopology

use of org.apache.kafka.streams.processor.internals.TopologyMetadata.Subtopology in project kafka by apache.

the class RepartitionTopics method computeRepartitionTopicConfig.

/**
 * @param clusterMetadata  cluster metadata, eg which topics exist on the brokers
 */
private Map<String, InternalTopicConfig> computeRepartitionTopicConfig(final Cluster clusterMetadata) {
    final Set<TopicsInfo> allTopicsInfo = new HashSet<>();
    final Map<String, InternalTopicConfig> allRepartitionTopicConfigs = new HashMap<>();
    for (final Map.Entry<String, Map<Subtopology, TopicsInfo>> topologyEntry : topologyMetadata.topologyToSubtopologyTopicsInfoMap().entrySet()) {
        final String topologyName = topologyMetadata.hasNamedTopologies() ? topologyEntry.getKey() : null;
        final Set<TopicsInfo> topicsInfoForTopology = new HashSet<>();
        final Set<String> missingSourceTopicsForTopology = new HashSet<>();
        final Map<String, InternalTopicConfig> repartitionTopicConfigsForTopology = new HashMap<>();
        for (final Map.Entry<Subtopology, TopicsInfo> subtopologyEntry : topologyEntry.getValue().entrySet()) {
            final TopicsInfo topicsInfo = subtopologyEntry.getValue();
            topicsInfoForTopology.add(topicsInfo);
            repartitionTopicConfigsForTopology.putAll(topicsInfo.repartitionSourceTopics.values().stream().collect(Collectors.toMap(InternalTopicConfig::name, topicConfig -> topicConfig)));
            final Set<String> missingSourceTopicsForSubtopology = computeMissingExternalSourceTopics(topicsInfo, clusterMetadata);
            missingSourceTopicsForTopology.addAll(missingSourceTopicsForSubtopology);
            if (!missingSourceTopicsForSubtopology.isEmpty()) {
                missingInputTopicsBySubtopology.put(subtopologyEntry.getKey(), missingSourceTopicsForSubtopology);
                log.error("Subtopology {} was missing source topics {} and will be excluded from the current assignment, " + "this can be due to the consumer client's metadata being stale or because they have " + "not been created yet. Please verify that you have created all input topics; if they " + "do exist, you just need to wait for the metadata to be updated, at which time a new " + "rebalance will be kicked off automatically and the topology will be retried at that time." + topologyName, missingSourceTopicsForTopology);
            }
        }
        if (missingSourceTopicsForTopology.isEmpty()) {
            allRepartitionTopicConfigs.putAll(repartitionTopicConfigsForTopology);
            allTopicsInfo.addAll(topicsInfoForTopology);
        } else {
            log.debug("Skipping repartition topic validation for entire topology {} due to missing source topics {}", topologyName, missingSourceTopicsForTopology);
        }
    }
    setRepartitionSourceTopicPartitionCount(allRepartitionTopicConfigs, allTopicsInfo, clusterMetadata);
    return allRepartitionTopicConfigs;
}

Example 5 with Subtopology

use of org.apache.kafka.streams.processor.internals.TopologyMetadata.Subtopology in project kafka by apache.

the class PartitionGrouper method partitionGroups.

/**
 * Generate tasks with the assigned topic partitions.
 *
 * @param topicGroups   group of topics that need to be joined together
 * @param metadata      metadata of the consuming cluster
 * @return The map from generated task ids to the assigned partitions
 */
public Map<TaskId, Set<TopicPartition>> partitionGroups(final Map<Subtopology, Set<String>> topicGroups, final Cluster metadata) {
    final Map<TaskId, Set<TopicPartition>> groups = new HashMap<>();
    for (final Map.Entry<Subtopology, Set<String>> entry : topicGroups.entrySet()) {
        final Subtopology subtopology = entry.getKey();
        final Set<String> topicGroup = entry.getValue();
        final int maxNumPartitions = maxNumPartitions(metadata, topicGroup);
        for (int partitionId = 0; partitionId < maxNumPartitions; partitionId++) {
            final Set<TopicPartition> group = new HashSet<>(topicGroup.size());
            for (final String topic : topicGroup) {
                final List<PartitionInfo> partitions = metadata.partitionsForTopic(topic);
                if (partitionId < partitions.size()) {
                    group.add(new TopicPartition(topic, partitionId));
                }
            }
            groups.put(new TaskId(subtopology.nodeGroupId, partitionId, subtopology.namedTopology), Collections.unmodifiableSet(group));
        }
    }
    return Collections.unmodifiableMap(groups);
}