Examples with TopicPartitionInfo org.apache.kafka.common.TopicPartitionInfo used on opensource projects

Example 16 with TopicPartitionInfo

use of org.apache.kafka.common.TopicPartitionInfo in project kafka by apache.

the class WorkerUtils method getMatchingTopicPartitions.

/**
 * Returns list of existing, not internal, topics/partitions that match given pattern and
 * where partitions are in range [startPartition, endPartition]
 * @param adminClient     AdminClient
 * @param topicRegex      Topic regular expression to match
 * @return                List of topic names
 * @throws Throwable      If failed to get list of existing topics
 */
static Collection<TopicPartition> getMatchingTopicPartitions(Admin adminClient, String topicRegex, int startPartition, int endPartition) throws Throwable {
    final Pattern topicNamePattern = Pattern.compile(topicRegex);
    // first get list of matching topics
    List<String> matchedTopics = new ArrayList<>();
    ListTopicsResult res = adminClient.listTopics(new ListTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT));
    Map<String, TopicListing> topicListingMap = res.namesToListings().get();
    for (Map.Entry<String, TopicListing> topicListingEntry : topicListingMap.entrySet()) {
        if (!topicListingEntry.getValue().isInternal() && topicNamePattern.matcher(topicListingEntry.getKey()).matches()) {
            matchedTopics.add(topicListingEntry.getKey());
        }
    }
    // create a list of topic/partitions
    List<TopicPartition> out = new ArrayList<>();
    DescribeTopicsResult topicsResult = adminClient.describeTopics(matchedTopics, new DescribeTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT));
    Map<String, TopicDescription> topicDescriptionMap = topicsResult.allTopicNames().get();
    for (TopicDescription desc : topicDescriptionMap.values()) {
        List<TopicPartitionInfo> partitions = desc.partitions();
        for (TopicPartitionInfo info : partitions) {
            if ((info.partition() >= startPartition) && (info.partition() <= endPartition)) {
                out.add(new TopicPartition(desc.name(), info.partition()));
            }
        }
    }
    return out;
}

Example 17 with TopicPartitionInfo

use of org.apache.kafka.common.TopicPartitionInfo in project kafka by apache.

the class TopicAdminTest method assertTopic.

protected void assertTopic(MockAdminClient admin, String topicName, int expectedPartitions, int expectedReplicas) {
    TopicDescription desc = null;
    try {
        desc = topicDescription(admin, topicName);
    } catch (Throwable t) {
        fail("Failed to find topic description for topic '" + topicName + "'");
    }
    assertEquals(expectedPartitions, desc.partitions().size());
    for (TopicPartitionInfo tp : desc.partitions()) {
        assertEquals(expectedReplicas, tp.replicas().size());
    }
}

Example 18 with TopicPartitionInfo

use of org.apache.kafka.common.TopicPartitionInfo in project kafka by apache.

the class TopicAdminTest method verifyingTopicCleanupPolicyShouldFailWhenTopicHasDeletePolicy.

@Test
public void verifyingTopicCleanupPolicyShouldFailWhenTopicHasDeletePolicy() {
    String topicName = "myTopic";
    Map<String, String> topicConfigs = Collections.singletonMap("cleanup.policy", "delete");
    Cluster cluster = createCluster(1);
    try (MockAdminClient mockAdminClient = new MockAdminClient(cluster.nodes(), cluster.nodeById(0))) {
        TopicPartitionInfo topicPartitionInfo = new TopicPartitionInfo(0, cluster.nodeById(0), cluster.nodes(), Collections.emptyList());
        mockAdminClient.addTopic(false, topicName, Collections.singletonList(topicPartitionInfo), topicConfigs);
        TopicAdmin admin = new TopicAdmin(null, mockAdminClient);
        ConfigException e = assertThrows(ConfigException.class, () -> admin.verifyTopicCleanupPolicyOnlyCompact("myTopic", "worker.topic", "purpose"));
        assertTrue(e.getMessage().contains("to guarantee consistency and durability"));
    }
}

Example 19 with TopicPartitionInfo

use of org.apache.kafka.common.TopicPartitionInfo in project kafka by apache.

the class TopicAdminTest method verifyingTopicCleanupPolicyShouldReturnTrueWhenTopicHasCorrectPolicy.

@Test
public void verifyingTopicCleanupPolicyShouldReturnTrueWhenTopicHasCorrectPolicy() {
    String topicName = "myTopic";
    Map<String, String> topicConfigs = Collections.singletonMap("cleanup.policy", "compact");
    Cluster cluster = createCluster(1);
    try (MockAdminClient mockAdminClient = new MockAdminClient(cluster.nodes(), cluster.nodeById(0))) {
        TopicPartitionInfo topicPartitionInfo = new TopicPartitionInfo(0, cluster.nodeById(0), cluster.nodes(), Collections.emptyList());
        mockAdminClient.addTopic(false, topicName, Collections.singletonList(topicPartitionInfo), topicConfigs);
        TopicAdmin admin = new TopicAdmin(null, mockAdminClient);
        boolean result = admin.verifyTopicCleanupPolicyOnlyCompact("myTopic", "worker.topic", "purpose");
        assertTrue(result);
    }
}

Example 20 with TopicPartitionInfo

use of org.apache.kafka.common.TopicPartitionInfo in project kafka by apache.

the class InternalTopicManagerTest method shouldNotThrowExceptionIfExistsWithDifferentReplication.

@Test
public void shouldNotThrowExceptionIfExistsWithDifferentReplication() {
    mockAdminClient.addTopic(false, topic1, Collections.singletonList(new TopicPartitionInfo(0, broker1, cluster, Collections.emptyList())), null);
    // attempt to create it again with replication 1
    final InternalTopicManager internalTopicManager2 = new InternalTopicManager(time, mockAdminClient, new StreamsConfig(config));
    final InternalTopicConfig internalTopicConfig = new RepartitionTopicConfig(topic1, Collections.emptyMap());
    internalTopicConfig.setNumberOfPartitions(1);
    internalTopicManager2.makeReady(Collections.singletonMap(topic1, internalTopicConfig));
}