Examples with KafkaConsumerGroupClient io.confluent.ksql.util.KafkaConsumerGroupClient used on opensource projects

Example 1 with KafkaConsumerGroupClient

use of io.confluent.ksql.util.KafkaConsumerGroupClient in project ksql by confluentinc.

the class KafkaTopicsList method build.

public static KafkaTopicsList build(String statementText, Collection<KsqlTopic> ksqlTopics, Map<String, TopicDescription> kafkaTopicDescriptions, KsqlConfig ksqlConfig, KafkaConsumerGroupClient consumerGroupClient) {
    Set<String> registeredNames = getRegisteredKafkaTopicNames(ksqlTopics);
    List<KafkaTopicInfo> kafkaTopicInfoList = new ArrayList<>();
    kafkaTopicDescriptions = new TreeMap<>(filterKsqlInternalTopics(kafkaTopicDescriptions, ksqlConfig));
    Map<String, List<Integer>> topicConsumersAndGroupCount = getTopicConsumerAndGroupCounts(consumerGroupClient);
    for (TopicDescription desp : kafkaTopicDescriptions.values()) {
        kafkaTopicInfoList.add(new KafkaTopicInfo(desp.name(), registeredNames.contains(desp.name()), desp.partitions().stream().map(partition -> partition.replicas().size()).collect(Collectors.toList()), topicConsumersAndGroupCount.getOrDefault(desp.name(), Arrays.asList(0, 0)).get(0), topicConsumersAndGroupCount.getOrDefault(desp.name(), Arrays.asList(0, 0)).get(1)));
    }
    return new KafkaTopicsList(statementText, kafkaTopicInfoList);
}

Example 2 with KafkaConsumerGroupClient

use of io.confluent.ksql.util.KafkaConsumerGroupClient in project ksql by confluentinc.

the class KafkaTopicsListTest method shouldBuildValidTopicList.

@Test
public void shouldBuildValidTopicList() {
    Collection<KsqlTopic> ksqlTopics = Collections.emptyList();
    // represent the full list of topics
    Map<String, TopicDescription> topicDescriptions = new HashMap<>();
    TopicPartitionInfo topicPartitionInfo = new TopicPartitionInfo(1, new Node(1, "", 8088), Collections.emptyList(), Collections.emptyList());
    topicDescriptions.put("test-topic", new TopicDescription("test-topic", false, Collections.singletonList(topicPartitionInfo)));
    /**
     * Return POJO for consumerGroupClient
     */
    TopicPartition topicPartition = new TopicPartition("test-topic", 1);
    KafkaConsumerGroupClientImpl.ConsumerSummary consumerSummary = new KafkaConsumerGroupClientImpl.ConsumerSummary("consumer-id");
    consumerSummary.addPartition(topicPartition);
    KafkaConsumerGroupClientImpl.ConsumerGroupSummary consumerGroupSummary = new KafkaConsumerGroupClientImpl.ConsumerGroupSummary();
    consumerGroupSummary.addConsumerSummary(consumerSummary);
    KafkaConsumerGroupClient consumerGroupClient = mock(KafkaConsumerGroupClient.class);
    expect(consumerGroupClient.listGroups()).andReturn(Collections.singletonList("test-topic"));
    expect(consumerGroupClient.describeConsumerGroup("test-topic")).andReturn(consumerGroupSummary);
    replay(consumerGroupClient);
    /**
     * Test
     */
    KafkaTopicsList topicsList = KafkaTopicsList.build("statement test", ksqlTopics, topicDescriptions, new KsqlConfig(Collections.EMPTY_MAP), consumerGroupClient);
    assertThat(topicsList.getTopics().size(), equalTo(1));
    KafkaTopicInfo first = topicsList.getTopics().iterator().next();
    assertThat(first.getConsumerGroupCount(), equalTo(1));
    assertThat(first.getConsumerCount(), equalTo(1));
    assertThat(first.getReplicaInfo().size(), equalTo(1));
}

Example 3 with KafkaConsumerGroupClient

use of io.confluent.ksql.util.KafkaConsumerGroupClient in project ksql by confluentinc.

the class KafkaTopicsList method getTopicConsumerAndGroupCounts.

/**
 * @return all topics with their associated consumerCount and consumerGroupCount
 */
private static Map<String, List<Integer>> getTopicConsumerAndGroupCounts(KafkaConsumerGroupClient consumerGroupClient) {
    List<String> consumerGroups = consumerGroupClient.listGroups();
    Map<String, AtomicInteger> topicConsumerCount = new HashMap<>();
    Map<String, Set<String>> topicConsumerGroupCount = new HashMap<>();
    for (String group : consumerGroups) {
        Collection<KafkaConsumerGroupClientImpl.ConsumerSummary> consumerSummaryList = consumerGroupClient.describeConsumerGroup(group).consumers();
        for (KafkaConsumerGroupClientImpl.ConsumerSummary consumerSummary : consumerSummaryList) {
            for (TopicPartition topicPartition : consumerSummary.partitions()) {
                topicConsumerCount.computeIfAbsent(topicPartition.topic(), k -> new AtomicInteger()).incrementAndGet();
                topicConsumerGroupCount.computeIfAbsent(topicPartition.topic(), k -> new HashSet<>()).add(group);
            }
        }
    }
    HashMap<String, List<Integer>> results = new HashMap<>();
    topicConsumerCount.forEach((k, v) -> {
        results.computeIfAbsent(k, v1 -> new ArrayList<>()).add(v.intValue());
        results.get(k).add(topicConsumerGroupCount.get(k).size());
    });
    return results;
}