Examples with AdminClient org.apache.kafka.clients.admin.AdminClient used on opensource projects

Example 71 with AdminClient

use of org.apache.kafka.clients.admin.AdminClient in project incubator-gobblin by apache.

the class Kafka1TopicProvisionTest method testLiveTopicPartitionCreationCount.

@Test(enabled = false)
public void testLiveTopicPartitionCreationCount() throws IOException, InterruptedException, ExecutionException {
    String liveClusterCount = System.getProperty("live.cluster.count");
    String liveZookeeper = System.getProperty("live.zookeeper");
    String liveBroker = System.getProperty("live.broker");
    String topic = System.getProperty("live.newtopic");
    String topicReplicationCount = System.getProperty("live.newtopic.replicationCount");
    String topicPartitionCount = System.getProperty("live.newtopic.partitionCount");
    if (StringUtils.isEmpty(liveClusterCount)) {
        Assert.assertTrue(true);
        return;
    }
    if (StringUtils.isEmpty(topicPartitionCount)) {
        int clusterCount = Integer.parseInt(liveClusterCount);
        clusterCount--;
        int partionCount = clusterCount / 2;
        topicReplicationCount = String.valueOf(clusterCount);
        topicPartitionCount = String.valueOf(partionCount);
    }
    Properties props = new Properties();
    // Setting Topic Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.REPLICATION_COUNT, topicReplicationCount);
    props.setProperty(KafkaWriterConfigurationKeys.PARTITION_COUNT, topicPartitionCount);
    props.setProperty(KafkaWriterConfigurationKeys.CLUSTER_ZOOKEEPER, liveZookeeper);
    // Setting Producer Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", liveBroker);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Kafka1DataWriter<String, String> kafka1DataWriter = new Kafka1DataWriter<>(props);
    int sessionTimeoutMs = 10 * 1000;
    int connectionTimeoutMs = 8 * 1000;
    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // createTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    ZkClient zkClient = new ZkClient(liveZookeeper, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
    boolean isSecureKafkaCluster = false;
    ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(liveZookeeper), isSecureKafkaCluster);
    Properties config = new Properties();
    config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, _kafkaTestHelper.getBootServersList());
    AdminClient adminClient = AdminClient.create(config);
    DescribeTopicsResult describer = adminClient.describeTopics(Collections.singletonList(topic));
    // Note: AdminUtils.fetchTopicMetadataFromZk is deprecated after 0.10.0. Please consider using AdminClient
    // to fetch topic config, or using ZKUtils.
    Assert.assertEquals(describer.values().get(topic).get().partitions().size(), Integer.parseInt(topicPartitionCount));
}