Examples with KafkaServer kafka.server.KafkaServer used on opensource projects

Example 1 with KafkaServer

use of kafka.server.KafkaServer in project deeplearning4j by deeplearning4j.

the class EmbeddedKafkaCluster method startup.

public void startup() {
    for (int i = 0; i < ports.size(); i++) {
        Integer port = ports.get(i);
        File logDir = TestUtils.constructTempDir("kafka-local");
        Properties properties = new Properties();
        properties.putAll(baseProperties);
        properties.setProperty("zookeeper.connect", zkConnection);
        properties.setProperty("broker.id", String.valueOf(i + 1));
        properties.setProperty("host.name", "localhost");
        properties.setProperty("port", Integer.toString(port));
        properties.setProperty("log.dir", logDir.getAbsolutePath());
        properties.setProperty("num.partitions", String.valueOf(1));
        properties.setProperty("auto.create.topics.enable", String.valueOf(Boolean.TRUE));
        System.out.println("EmbeddedKafkaCluster: local directory: " + logDir.getAbsolutePath());
        properties.setProperty("log.flush.interval.messages", String.valueOf(1));
        KafkaServer broker = startBroker(properties);
        brokers.add(broker);
        logDirs.add(logDir);
    }
}

Example 2 with KafkaServer

use of kafka.server.KafkaServer in project flink by apache.

the class KafkaTestEnvironmentImpl method getKafkaServer.

/**
	 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed)
	 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
    Properties kafkaProperties = new Properties();
    // properties have to be Strings
    kafkaProperties.put("advertised.host.name", KAFKA_HOST);
    kafkaProperties.put("broker.id", Integer.toString(brokerId));
    kafkaProperties.put("log.dir", tmpFolder.toString());
    kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
    kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
    kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
    // for CI stability, increase zookeeper session timeout
    kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
    kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
    if (additionalServerProperties != null) {
        kafkaProperties.putAll(additionalServerProperties);
    }
    final int numTries = 5;
    for (int i = 1; i <= numTries; i++) {
        int kafkaPort = NetUtils.getAvailablePort();
        kafkaProperties.put("port", Integer.toString(kafkaPort));
        //to support secure kafka cluster
        if (secureMode) {
            LOG.info("Adding Kafka secure configurations");
            kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
            kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
            kafkaProperties.putAll(getSecureProperties());
        }
        KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);
        try {
            scala.Option<String> stringNone = scala.Option.apply(null);
            KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
            server.startup();
            return server;
        } catch (KafkaException e) {
            if (e.getCause() instanceof BindException) {
                // port conflict, retry...
                LOG.info("Port conflict when starting Kafka Broker. Retrying...");
            } else {
                throw e;
            }
        }
    }
    throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}

Example 3 with KafkaServer

use of kafka.server.KafkaServer in project flink by apache.

the class KafkaTestEnvironmentImpl method shutdown.

@Override
public void shutdown() {
    for (KafkaServer broker : brokers) {
        if (broker != null) {
            broker.shutdown();
        }
    }
    brokers.clear();
    if (zookeeper != null) {
        try {
            zookeeper.stop();
            zookeeper.close();
        } catch (Exception e) {
            LOG.warn("ZK.stop() failed", e);
        }
        zookeeper = null;
    }
    if (tmpKafkaParent != null && tmpKafkaParent.exists()) {
        try {
            FileUtils.deleteDirectory(tmpKafkaParent);
        } catch (Exception e) {
        // ignore
        }
    }
    if (tmpZkDir != null && tmpZkDir.exists()) {
        try {
            FileUtils.deleteDirectory(tmpZkDir);
        } catch (Exception e) {
        // ignore
        }
    }
}

Example 4 with KafkaServer

use of kafka.server.KafkaServer in project spark-dataflow by cloudera.

the class EmbeddedKafkaCluster method startup.

public void startup() {
    for (int i = 0; i < ports.size(); i++) {
        Integer port = ports.get(i);
        File logDir = TestUtils.constructTempDir("kafka-local");
        Properties properties = new Properties();
        properties.putAll(baseProperties);
        properties.setProperty("zookeeper.connect", zkConnection);
        properties.setProperty("broker.id", String.valueOf(i + 1));
        properties.setProperty("host.name", "localhost");
        properties.setProperty("port", Integer.toString(port));
        properties.setProperty("log.dir", logDir.getAbsolutePath());
        properties.setProperty("log.flush.interval.messages", String.valueOf(1));
        KafkaServer broker = startBroker(properties);
        brokers.add(broker);
        logDirs.add(logDir);
    }
}

Example 5 with KafkaServer

use of kafka.server.KafkaServer in project spark-dataflow by cloudera.

the class EmbeddedKafkaCluster method startBroker.

private static KafkaServer startBroker(Properties props) {
    KafkaServer server = new KafkaServer(new KafkaConfig(props), new SystemTime());
    server.startup();
    return server;
}