Search in sources :

Example 16 with KafkaServer

use of kafka.server.KafkaServer in project apex-malhar by apache.

the class EmbeddedKafka method publish.

public void publish(String topic, List<String> messages) {
    Properties producerProps = new Properties();
    producerProps.setProperty("bootstrap.servers", BROKERHOST + ":" + BROKERPORT);
    producerProps.setProperty("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
    producerProps.setProperty("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    try (KafkaProducer<Integer, byte[]> producer = new KafkaProducer<>(producerProps)) {
        for (String message : messages) {
            ProducerRecord<Integer, byte[]> data = new ProducerRecord<>(topic, message.getBytes(StandardCharsets.UTF_8));
            producer.send(data);
        }
    }
    List<KafkaServer> servers = new ArrayList<KafkaServer>();
    servers.add(kafkaServer);
    TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 30000);
}
Also used : KafkaProducer(org.apache.kafka.clients.producer.KafkaProducer) KafkaServer(kafka.server.KafkaServer) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) ArrayList(java.util.ArrayList) Properties(java.util.Properties)

Example 17 with KafkaServer

use of kafka.server.KafkaServer in project druid by druid-io.

the class TestKafkaExtractionCluster method setUp.

@Before
public void setUp() throws Exception {
    zkTestServer = new TestingServer(-1, temporaryFolder.newFolder(), true);
    zkTestServer.start();
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            zkTestServer.stop();
        }
    });
    zkClient = new ZkClient(zkTestServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$);
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            zkClient.close();
        }
    });
    if (!zkClient.exists("/kafka")) {
        zkClient.create("/kafka", null, CreateMode.PERSISTENT);
    }
    log.info("---------------------------Started ZK---------------------------");
    final String zkKafkaPath = "/kafka";
    final Properties serverProperties = new Properties();
    serverProperties.putAll(kafkaProperties);
    serverProperties.put("broker.id", "0");
    serverProperties.put("log.dir", temporaryFolder.newFolder().getAbsolutePath());
    serverProperties.put("log.cleaner.enable", "true");
    serverProperties.put("host.name", "127.0.0.1");
    serverProperties.put("zookeeper.connect", zkTestServer.getConnectString() + zkKafkaPath);
    serverProperties.put("zookeeper.session.timeout.ms", "10000");
    serverProperties.put("zookeeper.sync.time.ms", "200");
    kafkaConfig = new KafkaConfig(serverProperties);
    final long time = DateTime.parse("2015-01-01").getMillis();
    kafkaServer = new KafkaServer(kafkaConfig, new Time() {

        @Override
        public long milliseconds() {
            return time;
        }

        @Override
        public long nanoseconds() {
            return milliseconds() * 1_000_000;
        }

        @Override
        public void sleep(long ms) {
            try {
                Thread.sleep(ms);
            } catch (InterruptedException e) {
                throw Throwables.propagate(e);
            }
        }
    });
    kafkaServer.startup();
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            kafkaServer.shutdown();
            kafkaServer.awaitShutdown();
        }
    });
    int sleepCount = 0;
    while (!kafkaServer.kafkaController().isActive()) {
        Thread.sleep(100);
        if (++sleepCount > 10) {
            throw new InterruptedException("Controller took to long to awaken");
        }
    }
    log.info("---------------------------Started Kafka Server---------------------------");
    final ZkClient zkClient = new ZkClient(zkTestServer.getConnectString() + zkKafkaPath, 10000, 10000, ZKStringSerializer$.MODULE$);
    try (final AutoCloseable autoCloseable = new AutoCloseable() {

        @Override
        public void close() throws Exception {
            if (zkClient.exists(zkKafkaPath)) {
                try {
                    zkClient.deleteRecursive(zkKafkaPath);
                } catch (org.I0Itec.zkclient.exception.ZkException ex) {
                    log.warn(ex, "error deleting %s zk node", zkKafkaPath);
                }
            }
            zkClient.close();
        }
    }) {
        final Properties topicProperties = new Properties();
        topicProperties.put("cleanup.policy", "compact");
        if (!AdminUtils.topicExists(zkClient, topicName)) {
            AdminUtils.createTopic(zkClient, topicName, 1, 1, topicProperties);
        }
        log.info("---------------------------Created topic---------------------------");
        Assert.assertTrue(AdminUtils.topicExists(zkClient, topicName));
    }
    final Properties kafkaProducerProperties = makeProducerProperties();
    final Producer<byte[], byte[]> producer = new Producer<>(new ProducerConfig(kafkaProducerProperties));
    try (final AutoCloseable autoCloseable = new AutoCloseable() {

        @Override
        public void close() throws Exception {
            producer.close();
        }
    }) {
        producer.send(new KeyedMessage<>(topicName, StringUtils.toUtf8("abcdefg"), StringUtils.toUtf8("abcdefg")));
    }
    System.setProperty("druid.extensions.searchCurrentClassloader", "false");
    injector = Initialization.makeInjectorWithModules(GuiceInjectors.makeStartupInjector(), ImmutableList.of(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bindConstant().annotatedWith(Names.named("serviceName")).to("test");
            binder.bindConstant().annotatedWith(Names.named("servicePort")).to(0);
        }
    }, // These injections fail under IntelliJ but are required for maven
    new NamespaceExtractionModule(), new KafkaExtractionNamespaceModule()));
    mapper = injector.getInstance(ObjectMapper.class);
    log.info("--------------------------- placed default item via producer ---------------------------");
    final Map<String, String> consumerProperties = new HashMap<>(kafkaProperties);
    consumerProperties.put("zookeeper.connect", zkTestServer.getConnectString() + zkKafkaPath);
    consumerProperties.put("zookeeper.session.timeout.ms", "10000");
    consumerProperties.put("zookeeper.sync.time.ms", "200");
    final KafkaLookupExtractorFactory kafkaLookupExtractorFactory = new KafkaLookupExtractorFactory(null, topicName, consumerProperties);
    factory = (KafkaLookupExtractorFactory) mapper.readValue(mapper.writeValueAsString(kafkaLookupExtractorFactory), LookupExtractorFactory.class);
    Assert.assertEquals(kafkaLookupExtractorFactory.getKafkaTopic(), factory.getKafkaTopic());
    Assert.assertEquals(kafkaLookupExtractorFactory.getKafkaProperties(), factory.getKafkaProperties());
    factory.start();
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            factory.close();
        }
    });
    log.info("--------------------------- started rename manager ---------------------------");
}
Also used : NamespaceExtractionModule(io.druid.server.lookup.namespace.NamespaceExtractionModule) HashMap(java.util.HashMap) Closeable(java.io.Closeable) DateTime(org.joda.time.DateTime) Time(kafka.utils.Time) Properties(java.util.Properties) Binder(com.google.inject.Binder) ProducerConfig(kafka.producer.ProducerConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) TestingServer(org.apache.curator.test.TestingServer) ZkClient(org.I0Itec.zkclient.ZkClient) IOException(java.io.IOException) KafkaServer(kafka.server.KafkaServer) Producer(kafka.javaapi.producer.Producer) Module(com.google.inject.Module) NamespaceExtractionModule(io.druid.server.lookup.namespace.NamespaceExtractionModule) KafkaConfig(kafka.server.KafkaConfig) Before(org.junit.Before)

Example 18 with KafkaServer

use of kafka.server.KafkaServer in project druid by druid-io.

the class TestBroker method start.

public void start() {
    final Properties props = new Properties();
    props.setProperty("zookeeper.connect", zookeeperConnect);
    props.setProperty("zookeeper.session.timeout.ms", "30000");
    props.setProperty("zookeeper.connection.timeout.ms", "30000");
    props.setProperty("log.dirs", directory.toString());
    props.setProperty("broker.id", String.valueOf(id));
    props.setProperty("port", String.valueOf(new Random().nextInt(9999) + 10000));
    props.putAll(brokerProps);
    final KafkaConfig config = new KafkaConfig(props);
    server = new KafkaServer(config, SystemTime$.MODULE$, Some.apply(String.format("TestingBroker[%d]-", id)));
    server.startup();
}
Also used : KafkaServer(kafka.server.KafkaServer) Random(java.util.Random) Properties(java.util.Properties) KafkaConfig(kafka.server.KafkaConfig)

Example 19 with KafkaServer

use of kafka.server.KafkaServer in project deeplearning4j by deeplearning4j.

the class EmbeddedKafkaCluster method startBroker.

private KafkaServer startBroker(Properties props) {
    List<KafkaMetricsReporter> kmrList = new ArrayList<>();
    Buffer<KafkaMetricsReporter> metricsList = scala.collection.JavaConversions.asScalaBuffer(kmrList);
    KafkaServer server = new KafkaServer(new KafkaConfig(props), new SystemTime(), Option.<String>empty(), metricsList);
    server.startup();
    return server;
}
Also used : KafkaServer(kafka.server.KafkaServer) ArrayList(java.util.ArrayList) KafkaMetricsReporter(kafka.metrics.KafkaMetricsReporter) KafkaConfig(kafka.server.KafkaConfig)

Example 20 with KafkaServer

use of kafka.server.KafkaServer in project camel by apache.

the class EmbeddedKafkaBroker method startBroker.

private KafkaServer startBroker(Properties props) {
    List<KafkaMetricsReporter> kmrList = new ArrayList<>();
    Buffer<KafkaMetricsReporter> metricsList = scala.collection.JavaConversions.asScalaBuffer(kmrList);
    KafkaServer server = new KafkaServer(new KafkaConfig(props), new SystemTime(), Option.<String>empty(), metricsList);
    server.startup();
    return server;
}
Also used : KafkaServer(kafka.server.KafkaServer) ArrayList(java.util.ArrayList) KafkaMetricsReporter(kafka.metrics.KafkaMetricsReporter) KafkaConfig(kafka.server.KafkaConfig)

Aggregations

KafkaServer (kafka.server.KafkaServer)28 Properties (java.util.Properties)16 KafkaConfig (kafka.server.KafkaConfig)13 ArrayList (java.util.ArrayList)9 File (java.io.File)5 KafkaMetricsReporter (kafka.metrics.KafkaMetricsReporter)5 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 Binder (com.google.inject.Binder)2 Module (com.google.inject.Module)2 IOException (java.io.IOException)2 BindException (java.net.BindException)2 URL (java.net.URL)2 KafkaException (kafka.common.KafkaException)2 Time (kafka.utils.Time)2 ZkClient (org.I0Itec.zkclient.ZkClient)2 ApplicationProperties (org.apache.atlas.ApplicationProperties)2 TestingServer (org.apache.curator.test.TestingServer)2 Before (org.junit.Before)2 NamespaceExtractionModule (io.druid.server.lookup.namespace.NamespaceExtractionModule)1 Closeable (java.io.Closeable)1