Examples with DescribeTopicsOptions org.apache.kafka.clients.admin.DescribeTopicsOptions used on opensource projects

Example 1 with DescribeTopicsOptions

use of org.apache.kafka.clients.admin.DescribeTopicsOptions in project apache-kafka-on-k8s by banzaicloud.

the class WorkerUtils method verifyTopics.

/**
 * Verifies that topics in 'topicsToVerify' list have the same number of partitions as
 * described in 'topicsInfo'
 * @param log                The logger to use
 * @param adminClient        AdminClient
 * @param topicsToVerify     List of topics to verify
 * @param topicsInfo         Map of topic name to topic description, which includes topics in
 *                           'topicsToVerify' list.
 * @throws RuntimeException  If one or more topics have different number of partitions than
 * described in 'topicsInfo'
 */
private static void verifyTopics(Logger log, AdminClient adminClient, Collection<String> topicsToVerify, Map<String, NewTopic> topicsInfo) throws Throwable {
    DescribeTopicsResult topicsResult = adminClient.describeTopics(topicsToVerify, new DescribeTopicsOptions().timeoutMs(CREATE_TOPICS_REQUEST_TIMEOUT));
    Map<String, TopicDescription> topicDescriptionMap = topicsResult.all().get();
    for (TopicDescription desc : topicDescriptionMap.values()) {
        // map will always contain the topic since all topics in 'topicsExists' are in given
        // 'topics' map
        int partitions = topicsInfo.get(desc.name()).numPartitions();
        if (desc.partitions().size() != partitions) {
            String str = "Topic '" + desc.name() + "' exists, but has " + desc.partitions().size() + " partitions, while requested " + " number of partitions is " + partitions;
            log.warn(str);
            throw new RuntimeException(str);
        }
    }
}

Example 2 with DescribeTopicsOptions

use of org.apache.kafka.clients.admin.DescribeTopicsOptions in project kafka by apache.

the class TopicAdmin method describeTopics.

/**
 * Attempt to fetch the descriptions of the given topics
 * Apache Kafka added support for describing topics in 0.10.0.0, so this method works as expected with that and later versions.
 * With brokers older than 0.10.0.0, this method is unable to describe topics and always returns an empty set.
 *
 * @param topics the topics to describe
 * @return a map of topic names to topic descriptions of the topics that were requested; never null but possibly empty
 * @throws RetriableException if a retriable error occurs, the operation takes too long, or the
 * thread is interrupted while attempting to perform this operation
 * @throws ConnectException if a non retriable error occurs
 */
public Map<String, TopicDescription> describeTopics(String... topics) {
    if (topics == null) {
        return Collections.emptyMap();
    }
    String bootstrapServers = bootstrapServers();
    String topicNameList = String.join(", ", topics);
    Map<String, KafkaFuture<TopicDescription>> newResults = admin.describeTopics(Arrays.asList(topics), new DescribeTopicsOptions()).topicNameValues();
    // Iterate over each future so that we can handle individual failures like when some topics don't exist
    Map<String, TopicDescription> existingTopics = new HashMap<>();
    newResults.forEach((topic, desc) -> {
        try {
            existingTopics.put(topic, desc.get());
        } catch (ExecutionException e) {
            Throwable cause = e.getCause();
            if (cause instanceof UnknownTopicOrPartitionException) {
                log.debug("Topic '{}' does not exist on the brokers at {}", topic, bootstrapServers);
                return;
            }
            if (cause instanceof ClusterAuthorizationException || cause instanceof TopicAuthorizationException) {
                String msg = String.format("Not authorized to describe topic(s) '%s' on the brokers %s", topicNameList, bootstrapServers);
                throw new ConnectException(msg, cause);
            }
            if (cause instanceof UnsupportedVersionException) {
                String msg = String.format("Unable to describe topic(s) '%s' since the brokers " + "at %s do not support the DescribeTopics API.", topicNameList, bootstrapServers);
                throw new ConnectException(msg, cause);
            }
            if (cause instanceof TimeoutException) {
                // Timed out waiting for the operation to complete
                throw new RetriableException("Timed out while describing topics '" + topicNameList + "'", cause);
            }
            throw new ConnectException("Error while attempting to describe topics '" + topicNameList + "'", e);
        } catch (InterruptedException e) {
            Thread.interrupted();
            throw new RetriableException("Interrupted while attempting to describe topics '" + topicNameList + "'", e);
        }
    });
    return existingTopics;
}

Example 3 with DescribeTopicsOptions

use of org.apache.kafka.clients.admin.DescribeTopicsOptions in project kafka by apache.

the class WorkerUtils method getMatchingTopicPartitions.

/**
 * Returns list of existing, not internal, topics/partitions that match given pattern and
 * where partitions are in range [startPartition, endPartition]
 * @param adminClient     AdminClient
 * @param topicRegex      Topic regular expression to match
 * @return                List of topic names
 * @throws Throwable      If failed to get list of existing topics
 */
static Collection<TopicPartition> getMatchingTopicPartitions(Admin adminClient, String topicRegex, int startPartition, int endPartition) throws Throwable {
    final Pattern topicNamePattern = Pattern.compile(topicRegex);
    // first get list of matching topics
    List<String> matchedTopics = new ArrayList<>();
    ListTopicsResult res = adminClient.listTopics(new ListTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT));
    Map<String, TopicListing> topicListingMap = res.namesToListings().get();
    for (Map.Entry<String, TopicListing> topicListingEntry : topicListingMap.entrySet()) {
        if (!topicListingEntry.getValue().isInternal() && topicNamePattern.matcher(topicListingEntry.getKey()).matches()) {
            matchedTopics.add(topicListingEntry.getKey());
        }
    }
    // create a list of topic/partitions
    List<TopicPartition> out = new ArrayList<>();
    DescribeTopicsResult topicsResult = adminClient.describeTopics(matchedTopics, new DescribeTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT));
    Map<String, TopicDescription> topicDescriptionMap = topicsResult.allTopicNames().get();
    for (TopicDescription desc : topicDescriptionMap.values()) {
        List<TopicPartitionInfo> partitions = desc.partitions();
        for (TopicPartitionInfo info : partitions) {
            if ((info.partition() >= startPartition) && (info.partition() <= endPartition)) {
                out.add(new TopicPartition(desc.name(), info.partition()));
            }
        }
    }
    return out;
}