Examples with BrokerService org.apache.twill.kafka.client.BrokerService used on opensource projects

Example 1 with BrokerService

use of org.apache.twill.kafka.client.BrokerService in project cdap by caskdata.

the class KafkaTester method before.

@Override
protected void before() throws Throwable {
    int kafkaPort = Networks.getRandomPort();
    Preconditions.checkState(kafkaPort > 0, "Failed to get random port.");
    int zkServerPort = Networks.getRandomPort();
    Preconditions.checkState(zkServerPort > 0, "Failed to get random port.");
    tmpFolder.create();
    zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).setPort(zkServerPort).build();
    zkServer.startAndWait();
    LOG.info("In memory ZK started on {}", zkServer.getConnectionStr());
    kafkaServer = new EmbeddedKafkaServer(generateKafkaConfig(kafkaPort));
    kafkaServer.startAndWait();
    initializeCConf(kafkaPort);
    injector = createInjector();
    zkClient = injector.getInstance(ZKClientService.class);
    zkClient.startAndWait();
    kafkaClient = injector.getInstance(KafkaClientService.class);
    kafkaClient.startAndWait();
    brokerService = injector.getInstance(BrokerService.class);
    brokerService.startAndWait();
    LOG.info("Waiting for Kafka server to startup...");
    waitForKafkaStartup();
    LOG.info("Started kafka server on port {}", kafkaPort);
}

Example 2 with BrokerService

use of org.apache.twill.kafka.client.BrokerService in project cdap by caskdata.

the class DistributedLogFramework method createService.

@Override
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(new Supplier<Service>() {

            @Override
            public Service get() {
                return new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(topic, pipelineSpec.getCheckpointPrefix()), brokerService, config);
            }
        }, retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, new Function<Service, ListenableFuture<State>>() {

                @Override
                public ListenableFuture<State> apply(Service service) {
                    return service.start();
                }
            }));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, new Function<Service, ListenableFuture<State>>() {

                @Override
                public ListenableFuture<State> apply(Service service) {
                    return service.stop();
                }
            }));
        }
    };
}

Example 3 with BrokerService

use of org.apache.twill.kafka.client.BrokerService in project cdap by caskdata.

the class KafkaClientModuleTest method testWithSharedZKClient.

@Test
public void testWithSharedZKClient() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new KafkaClientModule());
    // Get the shared zkclient and start it
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    int baseZKConns = getZKConnections();
    KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class);
    final BrokerService brokerService = injector.getInstance(BrokerService.class);
    // Start both kafka and broker services, it shouldn't affect the state of the shared zk client
    kafkaClientService.startAndWait();
    brokerService.startAndWait();
    // Shouldn't affect the shared zk client state
    Assert.assertTrue(zkClientService.isRunning());
    // It shouldn't increase the number of zk client connections
    Assert.assertEquals(baseZKConns, getZKConnections());
    // Make sure it is talking to Kafka.
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerService.getBrokers().iterator().hasNext();
        }
    }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Stop both, still shouldn't affect the state of the shared zk client
    kafkaClientService.stopAndWait();
    brokerService.stopAndWait();
    // Still shouldn't affect the shared zk client
    Assert.assertTrue(zkClientService.isRunning());
    // It still shouldn't increase the number of zk client connections
    Assert.assertEquals(baseZKConns, getZKConnections());
    zkClientService.stopAndWait();
}

Example 4 with BrokerService

use of org.apache.twill.kafka.client.BrokerService in project cdap by caskdata.

the class KafkaClientModuleTest method testWithDedicatedZKClient.

@Test
public void testWithDedicatedZKClient() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    // Set the zk quorum for the kafka client, expects it to create and start/stop it's own zk client service
    cConf.set(KafkaConstants.ConfigKeys.ZOOKEEPER_QUORUM, kafkaZKConnect);
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new KafkaClientModule());
    // Get the shared zkclient and start it
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    int baseZKConns = getZKConnections();
    KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class);
    final BrokerService brokerService = injector.getInstance(BrokerService.class);
    // Start the kafka client, it should increase the zk connections by 1
    kafkaClientService.startAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Start the broker service,
    // it shouldn't affect the zk connections, as it share the same zk client with kafka client
    brokerService.startAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Make sure it is talking to Kafka.
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerService.getBrokers().iterator().hasNext();
        }
    }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Shouldn't affect the shared zk client state
    Assert.assertTrue(zkClientService.isRunning());
    // Stop the broker service, it shouldn't affect the zk connections, as it is still used by the kafka client
    brokerService.stopAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Stop the kafka client, the zk connections should be reduced by 1
    kafkaClientService.stopAndWait();
    Assert.assertEquals(baseZKConns, getZKConnections());
    // Still shouldn't affect the shared zk client
    Assert.assertTrue(zkClientService.isRunning());
    zkClientService.stopAndWait();
}