Examples with TopicListing org.apache.kafka.clients.admin.TopicListing used on opensource projects

Example 1 with TopicListing

use of org.apache.kafka.clients.admin.TopicListing in project apache-kafka-on-k8s by banzaicloud.

the class ClientCompatibilityTest method testAdminClient.

void testAdminClient() throws Throwable {
    Properties adminProps = new Properties();
    adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, testConfig.bootstrapServer);
    try (final AdminClient client = AdminClient.create(adminProps)) {
        while (true) {
            Collection<Node> nodes = client.describeCluster().nodes().get();
            if (nodes.size() == testConfig.numClusterNodes) {
                break;
            } else if (nodes.size() > testConfig.numClusterNodes) {
                throw new KafkaException("Expected to see " + testConfig.numClusterNodes + " nodes, but saw " + nodes.size());
            }
            Thread.sleep(1);
            log.info("Saw only {} cluster nodes.  Waiting to see {}.", nodes.size(), testConfig.numClusterNodes);
        }
        tryFeature("createTopics", testConfig.createTopicsSupported, new Invoker() {

            @Override
            public void invoke() throws Throwable {
                try {
                    client.createTopics(Collections.singleton(new NewTopic("newtopic", 1, (short) 1))).all().get();
                } catch (ExecutionException e) {
                    throw e.getCause();
                }
            }
        }, new ResultTester() {

            @Override
            public void test() throws Throwable {
                while (true) {
                    try {
                        client.describeTopics(Collections.singleton("newtopic")).all().get();
                        break;
                    } catch (ExecutionException e) {
                        if (e.getCause() instanceof UnknownTopicOrPartitionException)
                            continue;
                        throw e;
                    }
                }
            }
        });
        while (true) {
            Collection<TopicListing> listings = client.listTopics().listings().get();
            if (!testConfig.createTopicsSupported)
                break;
            boolean foundNewTopic = false;
            for (TopicListing listing : listings) {
                if (listing.name().equals("newtopic")) {
                    if (listing.isInternal())
                        throw new KafkaException("Did not expect newtopic to be an internal topic.");
                    foundNewTopic = true;
                }
            }
            if (foundNewTopic)
                break;
            Thread.sleep(1);
            log.info("Did not see newtopic.  Retrying listTopics...");
        }
        tryFeature("describeAclsSupported", testConfig.describeAclsSupported, new Invoker() {

            @Override
            public void invoke() throws Throwable {
                try {
                    client.describeAcls(AclBindingFilter.ANY).values().get();
                } catch (ExecutionException e) {
                    if (e.getCause() instanceof SecurityDisabledException)
                        return;
                    throw e.getCause();
                }
            }
        });
    }
}

Example 2 with TopicListing

use of org.apache.kafka.clients.admin.TopicListing in project eventapis by kloiasoft.

the class Eventapis method main.

public static void main(String[] args) throws ExecutionException, InterruptedException {
    Map props = new HashMap<>();
    // list of host:port pairs used for establishing the initial connections
    // to the Kakfa cluster
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka-local:9092");
    // props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
    // JsonSerializer.class);
    // props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
    // JsonSerializer.class);
    // value to block, after which it will throw a TimeoutException
    props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 50000);
    AdminClient adminClient = AdminClient.create(props);
    adminClient.describeCluster();
    Collection<TopicListing> topicListings = adminClient.listTopics().listings().get();
    System.out.println(topicListings);
}

Example 3 with TopicListing

use of org.apache.kafka.clients.admin.TopicListing in project kafka by apache.

the class WorkerUtils method getMatchingTopicPartitions.

/**
 * Returns list of existing, not internal, topics/partitions that match given pattern and
 * where partitions are in range [startPartition, endPartition]
 * @param adminClient     AdminClient
 * @param topicRegex      Topic regular expression to match
 * @return                List of topic names
 * @throws Throwable      If failed to get list of existing topics
 */
static Collection<TopicPartition> getMatchingTopicPartitions(Admin adminClient, String topicRegex, int startPartition, int endPartition) throws Throwable {
    final Pattern topicNamePattern = Pattern.compile(topicRegex);
    // first get list of matching topics
    List<String> matchedTopics = new ArrayList<>();
    ListTopicsResult res = adminClient.listTopics(new ListTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT));
    Map<String, TopicListing> topicListingMap = res.namesToListings().get();
    for (Map.Entry<String, TopicListing> topicListingEntry : topicListingMap.entrySet()) {
        if (!topicListingEntry.getValue().isInternal() && topicNamePattern.matcher(topicListingEntry.getKey()).matches()) {
            matchedTopics.add(topicListingEntry.getKey());
        }
    }
    // create a list of topic/partitions
    List<TopicPartition> out = new ArrayList<>();
    DescribeTopicsResult topicsResult = adminClient.describeTopics(matchedTopics, new DescribeTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT));
    Map<String, TopicDescription> topicDescriptionMap = topicsResult.allTopicNames().get();
    for (TopicDescription desc : topicDescriptionMap.values()) {
        List<TopicPartitionInfo> partitions = desc.partitions();
        for (TopicPartitionInfo info : partitions) {
            if ((info.partition() >= startPartition) && (info.partition() <= endPartition)) {
                out.add(new TopicPartition(desc.name(), info.partition()));
            }
        }
    }
    return out;
}

Example 4 with TopicListing

use of org.apache.kafka.clients.admin.TopicListing in project kafka by apache.

the class ClientCompatibilityTest method testAdminClient.

void testAdminClient() throws Throwable {
    Properties adminProps = new Properties();
    adminProps.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, testConfig.bootstrapServer);
    try (final Admin client = Admin.create(adminProps)) {
        while (true) {
            Collection<Node> nodes = client.describeCluster().nodes().get();
            if (nodes.size() == testConfig.numClusterNodes) {
                break;
            } else if (nodes.size() > testConfig.numClusterNodes) {
                throw new KafkaException("Expected to see " + testConfig.numClusterNodes + " nodes, but saw " + nodes.size());
            }
            Thread.sleep(1);
            log.info("Saw only {} cluster nodes.  Waiting to see {}.", nodes.size(), testConfig.numClusterNodes);
        }
        testDescribeConfigsMethod(client);
        tryFeature("createTopics", testConfig.createTopicsSupported, () -> {
            try {
                client.createTopics(Collections.singleton(new NewTopic("newtopic", 1, (short) 1))).all().get();
            } catch (ExecutionException e) {
                throw e.getCause();
            }
        }, () -> createTopicsResultTest(client, Collections.singleton("newtopic")));
        while (true) {
            Collection<TopicListing> listings = client.listTopics().listings().get();
            if (!testConfig.createTopicsSupported)
                break;
            if (topicExists(listings, "newtopic"))
                break;
            Thread.sleep(1);
            log.info("Did not see newtopic.  Retrying listTopics...");
        }
        tryFeature("describeAclsSupported", testConfig.describeAclsSupported, () -> {
            try {
                client.describeAcls(AclBindingFilter.ANY).values().get();
            } catch (ExecutionException e) {
                if (e.getCause() instanceof SecurityDisabledException)
                    return;
                throw e.getCause();
            }
        });
    }
}

Example 5 with TopicListing

use of org.apache.kafka.clients.admin.TopicListing in project eventapis by kloiasoft.

the class ListTopicSchedule method runInternal.

@Override
boolean runInternal(StopWatch stopWatch) throws InterruptedException, ExecutionException {
    stopWatch.start("adminClient.listTopics()");
    Collection<String> topicNames = adminClient.listTopics().listings().get().stream().map(TopicListing::name).filter(this::shouldCollectEvent).collect(Collectors.toList());
    topicsMap.removeAll(new RemoveTopicPredicate(topicNames));
    DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(topicNames);
    describeTopicsResult.all().get().forEach((topic, topicDescription) -> topicsMap.executeOnKey(topic, new SetTopicPartitionsProcessor(topicDescription.partitions().stream().map(TopicPartitionInfo::partition).collect(Collectors.toList()))));
    metaMap.set(this.getName() + TopicServiceScheduler.LAST_SUCCESS_PREFIX, System.currentTimeMillis());
    log.debug("Topics:" + topicsMap.entrySet());
    log.debug(stopWatch.prettyPrint());
    return true;
}