Examples with KafkaServer kafka.server.KafkaServer used on opensource projects

Example 11 with KafkaServer

use of kafka.server.KafkaServer in project nd4j by deeplearning4j.

the class EmbeddedKafkaCluster method startup.

public void startup() {
    for (int i = 0; i < ports.size(); i++) {
        Integer port = ports.get(i);
        File logDir = TestUtils.constructTempDir("kafka-local");
        Properties properties = new Properties();
        properties.putAll(baseProperties);
        properties.setProperty("zookeeper.connect", zkConnection);
        properties.setProperty("broker.id", String.valueOf(i + 1));
        properties.setProperty("host.opName", "localhost");
        properties.setProperty("port", Integer.toString(port));
        properties.setProperty("log.dir", logDir.getAbsolutePath());
        properties.setProperty("num.partitions", String.valueOf(1));
        properties.setProperty("auto.create.topics.enable", String.valueOf(Boolean.TRUE));
        properties.setProperty("log.flush.interval.messages", String.valueOf(1));
        LOG.info("EmbeddedKafkaCluster: local directory: " + logDir.getAbsolutePath());
        KafkaServer broker = startBroker(properties);
        brokers.add(broker);
        logDirs.add(logDir);
    }
}

Example 12 with KafkaServer

use of kafka.server.KafkaServer in project drill by apache.

the class EmbeddedKafkaCluster method getBroker.

private static KafkaServer getBroker(Properties properties) {
    KafkaServer broker = new KafkaServer(new KafkaConfig(properties), Time.SYSTEM, Option.<String>apply("kafka"), false);
    broker.startup();
    return broker;
}

Example 13 with KafkaServer

use of kafka.server.KafkaServer in project kafka by apache.

the class IntegrationTestUtils method waitUntilMetadataIsPropagated.

private static void waitUntilMetadataIsPropagated(final List<KafkaServer> servers, final String topic, final int partition, final long timeout) throws InterruptedException {
    final String baseReason = String.format("Metadata for topic=%s partition=%d was not propagated to all brokers within %d ms. ", topic, partition, timeout);
    retryOnExceptionWithTimeout(timeout, () -> {
        final List<KafkaServer> emptyPartitionInfos = new ArrayList<>();
        final List<KafkaServer> invalidBrokerIds = new ArrayList<>();
        for (final KafkaServer server : servers) {
            final MetadataCache metadataCache = server.dataPlaneRequestProcessor().metadataCache();
            final Option<UpdateMetadataPartitionState> partitionInfo = metadataCache.getPartitionInfo(topic, partition);
            if (partitionInfo.isEmpty()) {
                emptyPartitionInfos.add(server);
                continue;
            }
            final UpdateMetadataPartitionState metadataPartitionState = partitionInfo.get();
            if (!Request.isValidBrokerId(metadataPartitionState.leader())) {
                invalidBrokerIds.add(server);
            }
        }
        final String reason = baseReason + ". Brokers without partition info: " + emptyPartitionInfos + ". Brokers with invalid broker id for partition leader: " + invalidBrokerIds;
        assertThat(reason, emptyPartitionInfos.isEmpty() && invalidBrokerIds.isEmpty());
    });
}

Example 14 with KafkaServer

use of kafka.server.KafkaServer in project druid by druid-io.

the class TestKafkaExtractionCluster method setUp.

@Before
public void setUp() throws Exception {
    zkServer = new TestingCluster(1);
    zkServer.start();
    closer.register(() -> {
        zkServer.stop();
    });
    kafkaServer = new KafkaServer(getBrokerProperties(), Time.SYSTEM, Some.apply(StringUtils.format("TestingBroker[%d]-", 1)), false);
    kafkaServer.startup();
    closer.register(() -> {
        kafkaServer.shutdown();
        kafkaServer.awaitShutdown();
    });
    log.info("---------------------------Started Kafka Broker ---------------------------");
    log.info("---------------------------Publish Messages to topic-----------------------");
    publishRecordsToKafka();
    System.setProperty("druid.extensions.searchCurrentClassloader", "false");
    injector = Initialization.makeInjectorWithModules(GuiceInjectors.makeStartupInjector(), ImmutableList.of(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bindConstant().annotatedWith(Names.named("serviceName")).to("test");
            binder.bindConstant().annotatedWith(Names.named("servicePort")).to(0);
            binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(-1);
        }
    }, // These injections fail under IntelliJ but are required for maven
    new NamespaceExtractionModule(), new KafkaExtractionNamespaceModule()));
    mapper = injector.getInstance(ObjectMapper.class);
    log.info("--------------------------- placed default item via producer ---------------------------");
    final Map<String, String> consumerProperties = getConsumerProperties();
    final KafkaLookupExtractorFactory kafkaLookupExtractorFactory = new KafkaLookupExtractorFactory(null, TOPIC_NAME, consumerProperties);
    factory = (KafkaLookupExtractorFactory) mapper.readValue(mapper.writeValueAsString(kafkaLookupExtractorFactory), LookupExtractorFactory.class);
    Assert.assertEquals(kafkaLookupExtractorFactory.getKafkaTopic(), factory.getKafkaTopic());
    Assert.assertEquals(kafkaLookupExtractorFactory.getKafkaProperties(), factory.getKafkaProperties());
    factory.start();
    closer.register(() -> factory.close());
    log.info("--------------------------- started rename manager ---------------------------");
}

Example 15 with KafkaServer

use of kafka.server.KafkaServer in project bagheera by mozilla-metrics.

the class ProducerTest method startServer.

private void startServer() {
    stopServer();
    Properties props = new Properties();
    props.setProperty("hostname", "localhost");
    props.setProperty("port", String.valueOf(KAFKA_BROKER_PORT));
    props.setProperty("brokerid", String.valueOf(KAFKA_BROKER_ID));
    props.setProperty("log.dir", KAFKA_DIR);
    props.setProperty("enable.zookeeper", "false");
    // flush every message.
    props.setProperty("log.flush.interval", "1");
    // flush every 1ms
    props.setProperty("log.default.flush.scheduler.interval.ms", "1");
    server = new KafkaServer(new KafkaConfig(props));
    server.startup();
}