Examples with TopicMetadata kafka.api.TopicMetadata used on opensource projects

Example 1 with TopicMetadata

use of kafka.api.TopicMetadata in project incubator-gobblin by apache.

the class Kafka09TopicProvisionTest method testTopicPartitionCreationCount.

@Test(enabled = false)
public void testTopicPartitionCreationCount() throws IOException, InterruptedException {
    String topic = "topicPartition4";
    int clusterCount = _kafkaTestHelper.getClusterCount();
    int partionCount = clusterCount / 2;
    int zkPort = _kafkaTestHelper.getZookeeperPort();
    Properties props = new Properties();
    // Setting Topic Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.REPLICATION_COUNT, String.valueOf(clusterCount));
    props.setProperty(KafkaWriterConfigurationKeys.PARTITION_COUNT, String.valueOf(partionCount));
    props.setProperty(KafkaWriterConfigurationKeys.CLUSTER_ZOOKEEPER, "localhost:" + zkPort);
    System.out.println(_kafkaTestHelper.getBootServersList());
    // Setting Producer Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaTestHelper.getBootServersList());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Kafka09DataWriter<String> kafka09DataWriter = new Kafka09DataWriter<String>(props);
    String zookeeperConnect = "localhost:" + _kafkaTestHelper.getZookeeperPort();
    int sessionTimeoutMs = 10 * 1000;
    int connectionTimeoutMs = 8 * 1000;
    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // createTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    ZkClient zkClient = new ZkClient(zookeeperConnect, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
    boolean isSecureKafkaCluster = false;
    ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect), isSecureKafkaCluster);
    TopicMetadata metaData = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
    Assert.assertEquals(metaData.partitionsMetadata().size(), partionCount);
}

Example 2 with TopicMetadata

use of kafka.api.TopicMetadata in project incubator-gobblin by apache.

the class Kafka09TopicProvisionTest method testLiveTopicPartitionCreationCount.

@Test(enabled = false)
public void testLiveTopicPartitionCreationCount() throws IOException, InterruptedException {
    String liveClusterCount = System.getProperty("live.cluster.count");
    String liveZookeeper = System.getProperty("live.zookeeper");
    String liveBroker = System.getProperty("live.broker");
    String topic = System.getProperty("live.newtopic");
    String topicReplicationCount = System.getProperty("live.newtopic.replicationCount");
    String topicPartitionCount = System.getProperty("live.newtopic.partitionCount");
    if (StringUtils.isEmpty(liveClusterCount)) {
        Assert.assertTrue(true);
        return;
    }
    if (StringUtils.isEmpty(topicPartitionCount)) {
        int clusterCount = Integer.parseInt(liveClusterCount);
        clusterCount--;
        int partionCount = clusterCount / 2;
        topicReplicationCount = String.valueOf(clusterCount);
        topicPartitionCount = String.valueOf(partionCount);
    }
    Properties props = new Properties();
    // Setting Topic Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.REPLICATION_COUNT, topicReplicationCount);
    props.setProperty(KafkaWriterConfigurationKeys.PARTITION_COUNT, topicPartitionCount);
    props.setProperty(KafkaWriterConfigurationKeys.CLUSTER_ZOOKEEPER, liveZookeeper);
    // Setting Producer Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", liveBroker);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Kafka09DataWriter<String> kafka09DataWriter = new Kafka09DataWriter<String>(props);
    int sessionTimeoutMs = 10 * 1000;
    int connectionTimeoutMs = 8 * 1000;
    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // createTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    ZkClient zkClient = new ZkClient(liveZookeeper, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
    boolean isSecureKafkaCluster = false;
    ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(liveZookeeper), isSecureKafkaCluster);
    TopicMetadata metaData = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
    Assert.assertEquals(metaData.partitionsMetadata().size(), Integer.parseInt(topicPartitionCount));
}

Example 3 with TopicMetadata

use of kafka.api.TopicMetadata in project nifi by apache.

the class KafkaUtils method retrievePartitionCountForTopic.

/**
 * Will retrieve the amount of partitions for a given Kafka topic.
 */
static int retrievePartitionCountForTopic(String zookeeperConnectionString, String topicName) {
    ZkClient zkClient = null;
    try {
        zkClient = new ZkClient(zookeeperConnectionString);
        zkClient.setZkSerializer(new ZkSerializer() {

            @Override
            public byte[] serialize(Object o) throws ZkMarshallingError {
                return ZKStringSerializer.serialize(o);
            }

            @Override
            public Object deserialize(byte[] bytes) throws ZkMarshallingError {
                return ZKStringSerializer.deserialize(bytes);
            }
        });
        scala.collection.Set<TopicMetadata> topicMetadatas = AdminUtils.fetchTopicMetadataFromZk(JavaConversions.asScalaSet(Collections.singleton(topicName)), zkClient);
        if (topicMetadatas != null && topicMetadatas.size() > 0) {
            return JavaConversions.asJavaSet(topicMetadatas).iterator().next().partitionsMetadata().size();
        } else {
            throw new IllegalStateException("Failed to get metadata for topic " + topicName);
        }
    } catch (Exception e) {
        throw new IllegalStateException("Failed to retrieve partitions for topic " + topicName, e);
    } finally {
        try {
            zkClient.close();
        } catch (Exception e2) {
        // ignore
        }
    }
}