Examples with TopicsInfo org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo used on opensource projects

Example 1 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class RepartitionTopicsTest method shouldThrowTaskAssignmentExceptionIfSourceTopicHasNoPartitionCount.

@Test
public void shouldThrowTaskAssignmentExceptionIfSourceTopicHasNoPartitionCount() {
    final RepartitionTopicConfig repartitionTopicConfigWithoutPartitionCount = new RepartitionTopicConfig(REPARTITION_WITHOUT_PARTITION_COUNT, TOPIC_CONFIG5);
    final TopicsInfo topicsInfo = new TopicsInfo(mkSet(REPARTITION_WITHOUT_PARTITION_COUNT), mkSet(SOURCE_TOPIC_NAME1), mkMap(mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)), Collections.emptyMap());
    expect(internalTopologyBuilder.subtopologyToTopicsInfo()).andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, topicsInfo), mkEntry(SUBTOPOLOGY_1, setupTopicInfoWithRepartitionTopicWithoutPartitionCount(repartitionTopicConfigWithoutPartitionCount))));
    expect(internalTopologyBuilder.copartitionGroups()).andReturn(Collections.emptyList());
    copartitionedTopicsEnforcer.enforce(eq(Collections.emptySet()), anyObject(), eq(clusterMetadata));
    expect(internalTopicManager.makeReady(mkMap(mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)))).andReturn(Collections.emptySet());
    setupClusterWithMissingPartitionCounts(mkSet(SOURCE_TOPIC_NAME1));
    replay(internalTopicManager, internalTopologyBuilder, clusterMetadata);
    final RepartitionTopics repartitionTopics = new RepartitionTopics(new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, "[test] ");
    final TaskAssignmentException exception = assertThrows(TaskAssignmentException.class, repartitionTopics::setup);
    assertThat(exception.getMessage(), is("No partition count found for source topic " + SOURCE_TOPIC_NAME1 + ", but it should have been."));
    assertThat(repartitionTopics.topologiesWithMissingInputTopics().isEmpty(), is(true));
    assertThat(repartitionTopics.missingSourceTopicExceptions().isEmpty(), is(true));
}

Example 2 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class RepartitionTopicsTest method shouldSetRepartitionTopicPartitionCountFromUpstreamExternalSourceTopic.

@Test
public void shouldSetRepartitionTopicPartitionCountFromUpstreamExternalSourceTopic() {
    final RepartitionTopicConfig repartitionTopicConfigWithoutPartitionCount = new RepartitionTopicConfig(REPARTITION_WITHOUT_PARTITION_COUNT, TOPIC_CONFIG5);
    final TopicsInfo topicsInfo = new TopicsInfo(mkSet(REPARTITION_TOPIC_NAME1, REPARTITION_WITHOUT_PARTITION_COUNT), mkSet(SOURCE_TOPIC_NAME1, REPARTITION_TOPIC_NAME2), mkMap(mkEntry(REPARTITION_TOPIC_NAME1, REPARTITION_TOPIC_CONFIG1), mkEntry(REPARTITION_TOPIC_NAME2, REPARTITION_TOPIC_CONFIG2), mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)), Collections.emptyMap());
    expect(internalTopologyBuilder.subtopologyToTopicsInfo()).andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, topicsInfo), mkEntry(SUBTOPOLOGY_1, setupTopicInfoWithRepartitionTopicWithoutPartitionCount(repartitionTopicConfigWithoutPartitionCount))));
    expect(internalTopologyBuilder.copartitionGroups()).andReturn(Collections.emptyList());
    copartitionedTopicsEnforcer.enforce(eq(Collections.emptySet()), anyObject(), eq(clusterMetadata));
    expect(internalTopicManager.makeReady(mkMap(mkEntry(REPARTITION_TOPIC_NAME1, REPARTITION_TOPIC_CONFIG1), mkEntry(REPARTITION_TOPIC_NAME2, REPARTITION_TOPIC_CONFIG2), mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)))).andReturn(Collections.emptySet());
    setupCluster();
    replay(internalTopicManager, internalTopologyBuilder, clusterMetadata);
    final RepartitionTopics repartitionTopics = new RepartitionTopics(new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, "[test] ");
    repartitionTopics.setup();
    verify(internalTopicManager, internalTopologyBuilder);
    final Map<TopicPartition, PartitionInfo> topicPartitionsInfo = repartitionTopics.topicPartitionsInfo();
    assertThat(topicPartitionsInfo.size(), is(9));
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 0);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 1);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 2);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 3);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME2, 0);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME2, 1);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_WITHOUT_PARTITION_COUNT, 0);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_WITHOUT_PARTITION_COUNT, 1);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_WITHOUT_PARTITION_COUNT, 2);
    assertThat(repartitionTopics.topologiesWithMissingInputTopics().isEmpty(), is(true));
    assertThat(repartitionTopics.missingSourceTopicExceptions().isEmpty(), is(true));
}

Example 3 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class RepartitionTopicsTest method shouldSetRepartitionTopicPartitionCountFromUpstreamInternalRepartitionSourceTopic.

@Test
public void shouldSetRepartitionTopicPartitionCountFromUpstreamInternalRepartitionSourceTopic() {
    final RepartitionTopicConfig repartitionTopicConfigWithoutPartitionCount = new RepartitionTopicConfig(REPARTITION_WITHOUT_PARTITION_COUNT, TOPIC_CONFIG5);
    final TopicsInfo topicsInfo = new TopicsInfo(mkSet(REPARTITION_TOPIC_NAME2, REPARTITION_WITHOUT_PARTITION_COUNT), mkSet(SOURCE_TOPIC_NAME1, REPARTITION_TOPIC_NAME1), mkMap(mkEntry(REPARTITION_TOPIC_NAME1, REPARTITION_TOPIC_CONFIG1), mkEntry(REPARTITION_TOPIC_NAME2, REPARTITION_TOPIC_CONFIG2), mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)), Collections.emptyMap());
    expect(internalTopologyBuilder.subtopologyToTopicsInfo()).andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, topicsInfo), mkEntry(SUBTOPOLOGY_1, setupTopicInfoWithRepartitionTopicWithoutPartitionCount(repartitionTopicConfigWithoutPartitionCount))));
    expect(internalTopologyBuilder.copartitionGroups()).andReturn(Collections.emptyList());
    copartitionedTopicsEnforcer.enforce(eq(Collections.emptySet()), anyObject(), eq(clusterMetadata));
    expect(internalTopicManager.makeReady(mkMap(mkEntry(REPARTITION_TOPIC_NAME1, REPARTITION_TOPIC_CONFIG1), mkEntry(REPARTITION_TOPIC_NAME2, REPARTITION_TOPIC_CONFIG2), mkEntry(REPARTITION_WITHOUT_PARTITION_COUNT, repartitionTopicConfigWithoutPartitionCount)))).andReturn(Collections.emptySet());
    setupCluster();
    replay(internalTopicManager, internalTopologyBuilder, clusterMetadata);
    final RepartitionTopics repartitionTopics = new RepartitionTopics(new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, "[test] ");
    repartitionTopics.setup();
    verify(internalTopicManager, internalTopologyBuilder);
    final Map<TopicPartition, PartitionInfo> topicPartitionsInfo = repartitionTopics.topicPartitionsInfo();
    assertThat(topicPartitionsInfo.size(), is(10));
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 0);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 1);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 2);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME1, 3);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME2, 0);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_TOPIC_NAME2, 1);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_WITHOUT_PARTITION_COUNT, 0);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_WITHOUT_PARTITION_COUNT, 1);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_WITHOUT_PARTITION_COUNT, 2);
    verifyRepartitionTopicPartitionInfo(topicPartitionsInfo, REPARTITION_WITHOUT_PARTITION_COUNT, 3);
    assertThat(repartitionTopics.topologiesWithMissingInputTopics().isEmpty(), is(true));
    assertThat(repartitionTopics.missingSourceTopicExceptions().isEmpty(), is(true));
}

Example 4 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class RepartitionTopicsTest method shouldNotSetupRepartitionTopicsWhenTopologyDoesNotContainAnyRepartitionTopics.

@Test
public void shouldNotSetupRepartitionTopicsWhenTopologyDoesNotContainAnyRepartitionTopics() {
    final TopicsInfo topicsInfo = new TopicsInfo(mkSet(SINK_TOPIC_NAME1), mkSet(SOURCE_TOPIC_NAME1), Collections.emptyMap(), Collections.emptyMap());
    expect(internalTopologyBuilder.subtopologyToTopicsInfo()).andReturn(mkMap(mkEntry(SUBTOPOLOGY_0, topicsInfo)));
    setupCluster();
    replay(internalTopicManager, internalTopologyBuilder, clusterMetadata);
    final RepartitionTopics repartitionTopics = new RepartitionTopics(new TopologyMetadata(internalTopologyBuilder, config), internalTopicManager, copartitionedTopicsEnforcer, clusterMetadata, "[test] ");
    repartitionTopics.setup();
    verify(internalTopicManager, internalTopologyBuilder);
    final Map<TopicPartition, PartitionInfo> topicPartitionsInfo = repartitionTopics.topicPartitionsInfo();
    assertThat(topicPartitionsInfo, is(Collections.emptyMap()));
    assertThat(repartitionTopics.topologiesWithMissingInputTopics().isEmpty(), is(true));
    assertThat(repartitionTopics.missingSourceTopicExceptions().isEmpty(), is(true));
}

Example 5 with TopicsInfo

use of org.apache.kafka.streams.processor.internals.InternalTopologyBuilder.TopicsInfo in project kafka by apache.

the class RepartitionTopics method computeRepartitionTopicConfig.

/**
 * @param clusterMetadata  cluster metadata, eg which topics exist on the brokers
 */
private Map<String, InternalTopicConfig> computeRepartitionTopicConfig(final Cluster clusterMetadata) {
    final Set<TopicsInfo> allTopicsInfo = new HashSet<>();
    final Map<String, InternalTopicConfig> allRepartitionTopicConfigs = new HashMap<>();
    for (final Map.Entry<String, Map<Subtopology, TopicsInfo>> topologyEntry : topologyMetadata.topologyToSubtopologyTopicsInfoMap().entrySet()) {
        final String topologyName = topologyMetadata.hasNamedTopologies() ? topologyEntry.getKey() : null;
        final Set<TopicsInfo> topicsInfoForTopology = new HashSet<>();
        final Set<String> missingSourceTopicsForTopology = new HashSet<>();
        final Map<String, InternalTopicConfig> repartitionTopicConfigsForTopology = new HashMap<>();
        for (final Map.Entry<Subtopology, TopicsInfo> subtopologyEntry : topologyEntry.getValue().entrySet()) {
            final TopicsInfo topicsInfo = subtopologyEntry.getValue();
            topicsInfoForTopology.add(topicsInfo);
            repartitionTopicConfigsForTopology.putAll(topicsInfo.repartitionSourceTopics.values().stream().collect(Collectors.toMap(InternalTopicConfig::name, topicConfig -> topicConfig)));
            final Set<String> missingSourceTopicsForSubtopology = computeMissingExternalSourceTopics(topicsInfo, clusterMetadata);
            missingSourceTopicsForTopology.addAll(missingSourceTopicsForSubtopology);
            if (!missingSourceTopicsForSubtopology.isEmpty()) {
                missingInputTopicsBySubtopology.put(subtopologyEntry.getKey(), missingSourceTopicsForSubtopology);
                log.error("Subtopology {} was missing source topics {} and will be excluded from the current assignment, " + "this can be due to the consumer client's metadata being stale or because they have " + "not been created yet. Please verify that you have created all input topics; if they " + "do exist, you just need to wait for the metadata to be updated, at which time a new " + "rebalance will be kicked off automatically and the topology will be retried at that time." + topologyName, missingSourceTopicsForTopology);
            }
        }
        if (missingSourceTopicsForTopology.isEmpty()) {
            allRepartitionTopicConfigs.putAll(repartitionTopicConfigsForTopology);
            allTopicsInfo.addAll(topicsInfoForTopology);
        } else {
            log.debug("Skipping repartition topic validation for entire topology {} due to missing source topics {}", topologyName, missingSourceTopicsForTopology);
        }
    }
    setRepartitionSourceTopicPartitionCount(allRepartitionTopicConfigs, allTopicsInfo, clusterMetadata);
    return allRepartitionTopicConfigs;
}