Examples with ZkUtils kafka.utils.ZkUtils used on opensource projects

Example 11 with ZkUtils

use of kafka.utils.ZkUtils in project cruise-control by linkedin.

the class KafkaSampleStore method ensureTopicCreated.

private void ensureTopicCreated(Map<String, ?> config) {
    ZkUtils zkUtils = createZkUtils(config);
    Map<String, List<PartitionInfo>> topics = _consumers.get(0).listTopics();
    long windowMs = Long.parseLong((String) config.get(KafkaCruiseControlConfig.METRICS_WINDOW_MS_CONFIG));
    int numWindows = Integer.parseInt((String) config.get(KafkaCruiseControlConfig.NUM_METRICS_WINDOWS_CONFIG));
    long retentionMs = (numWindows * ADDITIONAL_WINDOW_TO_RETAIN_FACTOR) * windowMs;
    Properties props = new Properties();
    props.setProperty(LogConfig.RetentionMsProp(), Long.toString(retentionMs));
    props.setProperty(LogConfig.CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY);
    int numberOfBrokersInCluster = zkUtils.getAllBrokersInCluster().size();
    if (numberOfBrokersInCluster == 0) {
        throw new IllegalStateException("Kafka cluster has no alive brokers.");
    }
    int replicationFactor = Math.min(2, numberOfBrokersInCluster);
    if (!topics.containsKey(_partitionMetricSampleStoreTopic)) {
        AdminUtils.createTopic(zkUtils, _partitionMetricSampleStoreTopic, 32, replicationFactor, props, RackAwareMode.Safe$.MODULE$);
    } else {
        AdminUtils.changeTopicConfig(zkUtils, _partitionMetricSampleStoreTopic, props);
    }
    if (!topics.containsKey(_brokerMetricSampleStoreTopic)) {
        AdminUtils.createTopic(zkUtils, _brokerMetricSampleStoreTopic, 32, replicationFactor, props, RackAwareMode.Safe$.MODULE$);
    } else {
        AdminUtils.changeTopicConfig(zkUtils, _brokerMetricSampleStoreTopic, props);
    }
    KafkaCruiseControlUtils.closeZkUtilsWithTimeout(zkUtils, 10000);
}

Example 12 with ZkUtils

use of kafka.utils.ZkUtils in project cruise-control by linkedin.

the class Executor method startExecution.

/**
 * Pause the load monitor and kick off the execution.
 *
 * @param loadMonitor Load monitor.
 */
private void startExecution(LoadMonitor loadMonitor) {
    // Pause the metric sampling to avoid the loss of accuracy during execution.
    loadMonitor.pauseMetricSampling();
    ZkUtils zkUtils = ZkUtils.apply(_zkConnect, ZK_SESSION_TIMEOUT, ZK_CONNECTION_TIMEOUT, IS_ZK_SECURITY_ENABLED);
    try {
        if (!ExecutorUtils.partitionsBeingReassigned(zkUtils).isEmpty()) {
            _executionTaskManager.clear();
            // Note that in case there is an ongoing partition reassignment, we do not unpause metric sampling.
            throw new IllegalStateException("There are ongoing partition reassignments.");
        }
        _hasOngoingExecution = true;
        _stopRequested.set(false);
        _proposalExecutor.submit(new ProposalExecutionRunnable(loadMonitor));
    } finally {
        KafkaCruiseControlUtils.closeZkUtilsWithTimeout(zkUtils, ZK_UTILS_CLOSE_TIMEOUT_MS);
    }
}

Example 13 with ZkUtils

use of kafka.utils.ZkUtils in project ksql by confluentinc.

the class KafkaEmbedded method createTopic.

/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(String topic, int partitions, int replication, Properties topicConfig) {
    log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }", topic, partitions, replication, topicConfig);
    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // registerTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    ZkClient zkClient = new ZkClient(zookeeperConnect(), DEFAULT_ZK_SESSION_TIMEOUT_MS, DEFAULT_ZK_CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), false);
    AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
    zkClient.close();
}

Example 14 with ZkUtils

use of kafka.utils.ZkUtils in project incubator-gobblin by apache.

the class Kafka09TopicProvisionTest method testTopicPartitionCreationCount.

@Test(enabled = false)
public void testTopicPartitionCreationCount() throws IOException, InterruptedException {
    String topic = "topicPartition4";
    int clusterCount = _kafkaTestHelper.getClusterCount();
    int partionCount = clusterCount / 2;
    int zkPort = _kafkaTestHelper.getZookeeperPort();
    Properties props = new Properties();
    // Setting Topic Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.REPLICATION_COUNT, String.valueOf(clusterCount));
    props.setProperty(KafkaWriterConfigurationKeys.PARTITION_COUNT, String.valueOf(partionCount));
    props.setProperty(KafkaWriterConfigurationKeys.CLUSTER_ZOOKEEPER, "localhost:" + zkPort);
    System.out.println(_kafkaTestHelper.getBootServersList());
    // Setting Producer Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaTestHelper.getBootServersList());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Kafka09DataWriter<String> kafka09DataWriter = new Kafka09DataWriter<String>(props);
    String zookeeperConnect = "localhost:" + _kafkaTestHelper.getZookeeperPort();
    int sessionTimeoutMs = 10 * 1000;
    int connectionTimeoutMs = 8 * 1000;
    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // createTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    ZkClient zkClient = new ZkClient(zookeeperConnect, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
    boolean isSecureKafkaCluster = false;
    ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect), isSecureKafkaCluster);
    TopicMetadata metaData = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
    Assert.assertEquals(metaData.partitionsMetadata().size(), partionCount);
}

Example 15 with ZkUtils

use of kafka.utils.ZkUtils in project incubator-gobblin by apache.

the class Kafka09TopicProvisionTest method testLiveTopicPartitionCreationCount.

@Test(enabled = false)
public void testLiveTopicPartitionCreationCount() throws IOException, InterruptedException {
    String liveClusterCount = System.getProperty("live.cluster.count");
    String liveZookeeper = System.getProperty("live.zookeeper");
    String liveBroker = System.getProperty("live.broker");
    String topic = System.getProperty("live.newtopic");
    String topicReplicationCount = System.getProperty("live.newtopic.replicationCount");
    String topicPartitionCount = System.getProperty("live.newtopic.partitionCount");
    if (StringUtils.isEmpty(liveClusterCount)) {
        Assert.assertTrue(true);
        return;
    }
    if (StringUtils.isEmpty(topicPartitionCount)) {
        int clusterCount = Integer.parseInt(liveClusterCount);
        clusterCount--;
        int partionCount = clusterCount / 2;
        topicReplicationCount = String.valueOf(clusterCount);
        topicPartitionCount = String.valueOf(partionCount);
    }
    Properties props = new Properties();
    // Setting Topic Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.REPLICATION_COUNT, topicReplicationCount);
    props.setProperty(KafkaWriterConfigurationKeys.PARTITION_COUNT, topicPartitionCount);
    props.setProperty(KafkaWriterConfigurationKeys.CLUSTER_ZOOKEEPER, liveZookeeper);
    // Setting Producer Properties
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", liveBroker);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Kafka09DataWriter<String> kafka09DataWriter = new Kafka09DataWriter<String>(props);
    int sessionTimeoutMs = 10 * 1000;
    int connectionTimeoutMs = 8 * 1000;
    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // createTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    ZkClient zkClient = new ZkClient(liveZookeeper, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
    boolean isSecureKafkaCluster = false;
    ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(liveZookeeper), isSecureKafkaCluster);
    TopicMetadata metaData = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
    Assert.assertEquals(metaData.partitionsMetadata().size(), Integer.parseInt(topicPartitionCount));
}