Examples with KafkaConfig kafka.server.KafkaConfig used on opensource projects

Example 6 with KafkaConfig

use of kafka.server.KafkaConfig in project flink by apache.

the class KafkaTestEnvironmentImpl method getKafkaServer.

/**
	 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed)
	 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
    Properties kafkaProperties = new Properties();
    // properties have to be Strings
    kafkaProperties.put("advertised.host.name", KAFKA_HOST);
    kafkaProperties.put("broker.id", Integer.toString(brokerId));
    kafkaProperties.put("log.dir", tmpFolder.toString());
    kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
    kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
    kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
    // for CI stability, increase zookeeper session timeout
    kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
    kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
    if (additionalServerProperties != null) {
        kafkaProperties.putAll(additionalServerProperties);
    }
    final int numTries = 5;
    for (int i = 1; i <= numTries; i++) {
        int kafkaPort = NetUtils.getAvailablePort();
        kafkaProperties.put("port", Integer.toString(kafkaPort));
        //to support secure kafka cluster
        if (secureMode) {
            LOG.info("Adding Kafka secure configurations");
            kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
            kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
            kafkaProperties.putAll(getSecureProperties());
        }
        KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);
        try {
            scala.Option<String> stringNone = scala.Option.apply(null);
            KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
            server.startup();
            return server;
        } catch (KafkaException e) {
            if (e.getCause() instanceof BindException) {
                // port conflict, retry...
                LOG.info("Port conflict when starting Kafka Broker. Retrying...");
            } else {
                throw e;
            }
        }
    }
    throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}

Example 7 with KafkaConfig

use of kafka.server.KafkaConfig in project ignite by apache.

the class TestKafkaBroker method setupKafkaServer.

/**
     * Sets up test Kafka broker.
     *
     * @throws IOException If failed.
     */
private void setupKafkaServer() throws IOException {
    kafkaCfg = new KafkaConfig(getKafkaConfig());
    kafkaSrv = TestUtils.createServer(kafkaCfg, SystemTime$.MODULE$);
    kafkaSrv.startup();
}

Example 8 with KafkaConfig

use of kafka.server.KafkaConfig in project phoenix by apache.

the class PhoenixConsumerIT method setUp.

@Before
public void setUp() throws IOException, SQLException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    String zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);
    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
    kafkaServer.startup();
    // create topic
    AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties());
    pConsumer = new PhoenixConsumer();
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    conn = DriverManager.getConnection(getUrl(), props);
}

Example 9 with KafkaConfig

use of kafka.server.KafkaConfig in project heron by twitter.

the class KafkaTestBroker method buildKafkaConfig.

private kafka.server.KafkaConfig buildKafkaConfig(String zookeeperConnectionString) {
    Properties p = new Properties();
    p.setProperty("zookeeper.connect", zookeeperConnectionString);
    p.setProperty("broker.id", "0");
    p.setProperty("port", "" + port);
    p.setProperty("log.dirs", logDir.getAbsolutePath());
    return new KafkaConfig(p);
}

Example 10 with KafkaConfig

use of kafka.server.KafkaConfig in project spark-dataflow by cloudera.

the class EmbeddedKafkaCluster method startBroker.

private static KafkaServer startBroker(Properties props) {
    KafkaServer server = new KafkaServer(new KafkaConfig(props), new SystemTime());
    server.startup();
    return server;
}