Examples with KafkaClientService org.apache.twill.kafka.client.KafkaClientService used on opensource projects

Example 1 with KafkaClientService

use of org.apache.twill.kafka.client.KafkaClientService in project cdap by caskdata.

the class SparkRuntimeContextProvider method createIfNotExists.

/**
   * Creates a singleton {@link SparkRuntimeContext}.
   * It has assumption on file location that are localized by the SparkRuntimeService.
   */
private static synchronized SparkRuntimeContext createIfNotExists() {
    if (sparkRuntimeContext != null) {
        return sparkRuntimeContext;
    }
    try {
        CConfiguration cConf = createCConf();
        Configuration hConf = createHConf();
        SparkRuntimeContextConfig contextConfig = new SparkRuntimeContextConfig(hConf);
        // Should be yarn only and only for executor node, not the driver node.
        Preconditions.checkState(!contextConfig.isLocal() && Boolean.parseBoolean(System.getenv("SPARK_YARN_MODE")), "SparkContextProvider.getSparkContext should only be called in Spark executor process.");
        // Create the program
        Program program = createProgram(cConf, contextConfig);
        Injector injector = createInjector(cConf, hConf, contextConfig.getProgramId(), contextConfig.getProgramOptions());
        final Service logAppenderService = new LogAppenderService(injector.getInstance(LogAppenderInitializer.class), contextConfig.getProgramOptions());
        final ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
        final KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class);
        final MetricsCollectionService metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
        final StreamCoordinatorClient streamCoordinatorClient = injector.getInstance(StreamCoordinatorClient.class);
        // Use the shutdown hook to shutdown services, since this class should only be loaded from System classloader
        // of the spark executor, hence there should be exactly one instance only.
        // The problem with not shutting down nicely is that some logs/metrics might be lost
        Services.chainStart(logAppenderService, zkClientService, kafkaClientService, metricsCollectionService, streamCoordinatorClient);
        Runtime.getRuntime().addShutdownHook(new Thread() {

            @Override
            public void run() {
                // The logger may already been shutdown. Use System.out/err instead
                System.out.println("Shutting SparkClassLoader services");
                Future<List<ListenableFuture<Service.State>>> future = Services.chainStop(logAppenderService, streamCoordinatorClient, metricsCollectionService, kafkaClientService, zkClientService);
                try {
                    List<ListenableFuture<Service.State>> futures = future.get(5, TimeUnit.SECONDS);
                    System.out.println("SparkClassLoader services shutdown completed: " + futures);
                } catch (Exception e) {
                    System.err.println("Exception when shutting down services");
                    e.printStackTrace(System.err);
                }
            }
        });
        // Constructor the DatasetFramework
        DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
        WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, contextConfig.getApplicationSpecification());
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions()));
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(cConf, contextConfig, program.getClassLoader());
        // Create the context object
        sparkRuntimeContext = new SparkRuntimeContext(contextConfig.getConfiguration(), program, contextConfig.getProgramOptions(), cConf, getHostname(), injector.getInstance(TransactionSystemClient.class), programDatasetFramework, injector.getInstance(DiscoveryServiceClient.class), metricsCollectionService, injector.getInstance(StreamAdmin.class), contextConfig.getWorkflowProgramInfo(), pluginInstantiator, injector.getInstance(SecureStore.class), injector.getInstance(SecureStoreManager.class), injector.getInstance(AuthorizationEnforcer.class), injector.getInstance(AuthenticationContext.class), injector.getInstance(MessagingService.class));
        LoggingContextAccessor.setLoggingContext(sparkRuntimeContext.getLoggingContext());
        return sparkRuntimeContext;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 2 with KafkaClientService

use of org.apache.twill.kafka.client.KafkaClientService in project cdap by caskdata.

the class KafkaClientModuleTest method testWithDedicatedZKClient.

@Test
public void testWithDedicatedZKClient() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    // Set the zk quorum for the kafka client, expects it to create and start/stop it's own zk client service
    cConf.set(KafkaConstants.ConfigKeys.ZOOKEEPER_QUORUM, kafkaZKConnect);
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new KafkaClientModule());
    // Get the shared zkclient and start it
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    int baseZKConns = getZKConnections();
    KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class);
    final BrokerService brokerService = injector.getInstance(BrokerService.class);
    // Start the kafka client, it should increase the zk connections by 1
    kafkaClientService.startAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Start the broker service,
    // it shouldn't affect the zk connections, as it share the same zk client with kafka client
    brokerService.startAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Make sure it is talking to Kafka.
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerService.getBrokers().iterator().hasNext();
        }
    }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Shouldn't affect the shared zk client state
    Assert.assertTrue(zkClientService.isRunning());
    // Stop the broker service, it shouldn't affect the zk connections, as it is still used by the kafka client
    brokerService.stopAndWait();
    Assert.assertEquals(baseZKConns + 1, getZKConnections());
    // Stop the kafka client, the zk connections should be reduced by 1
    kafkaClientService.stopAndWait();
    Assert.assertEquals(baseZKConns, getZKConnections());
    // Still shouldn't affect the shared zk client
    Assert.assertTrue(zkClientService.isRunning());
    zkClientService.stopAndWait();
}

Example 3 with KafkaClientService

use of org.apache.twill.kafka.client.KafkaClientService in project cdap by caskdata.

the class KafkaClientModuleTest method testWithSharedZKClient.

@Test
public void testWithSharedZKClient() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.Zookeeper.QUORUM, zkServer.getConnectionStr());
    Injector injector = Guice.createInjector(new ConfigModule(cConf), new ZKClientModule(), new KafkaClientModule());
    // Get the shared zkclient and start it
    ZKClientService zkClientService = injector.getInstance(ZKClientService.class);
    zkClientService.startAndWait();
    int baseZKConns = getZKConnections();
    KafkaClientService kafkaClientService = injector.getInstance(KafkaClientService.class);
    final BrokerService brokerService = injector.getInstance(BrokerService.class);
    // Start both kafka and broker services, it shouldn't affect the state of the shared zk client
    kafkaClientService.startAndWait();
    brokerService.startAndWait();
    // Shouldn't affect the shared zk client state
    Assert.assertTrue(zkClientService.isRunning());
    // It shouldn't increase the number of zk client connections
    Assert.assertEquals(baseZKConns, getZKConnections());
    // Make sure it is talking to Kafka.
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerService.getBrokers().iterator().hasNext();
        }
    }, 5L, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Stop both, still shouldn't affect the state of the shared zk client
    kafkaClientService.stopAndWait();
    brokerService.stopAndWait();
    // Still shouldn't affect the shared zk client
    Assert.assertTrue(zkClientService.isRunning());
    // It still shouldn't increase the number of zk client connections
    Assert.assertEquals(baseZKConns, getZKConnections());
    zkClientService.stopAndWait();
}

Example 4 with KafkaClientService

use of org.apache.twill.kafka.client.KafkaClientService in project cdap by caskdata.

the class MetricsProcessorServiceTest method testMetricsProcessor.

@Test
public void testMetricsProcessor() throws Exception {
    injector.getInstance(TransactionManager.class).startAndWait();
    injector.getInstance(DatasetOpExecutor.class).startAndWait();
    injector.getInstance(DatasetService.class).startAndWait();
    zkServer = InMemoryZKServer.builder().build();
    zkServer.startAndWait();
    Properties kafkaConfig = generateKafkaConfig(tmpFolder1);
    EmbeddedKafkaServer kafkaServer = new EmbeddedKafkaServer(kafkaConfig);
    kafkaServer.startAndWait();
    ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
    zkClient.startAndWait();
    KafkaClientService kafkaClient = new ZKKafkaClientService(zkClient);
    kafkaClient.startAndWait();
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    Set<Integer> partitions = new HashSet<>();
    for (int i = 0; i < PARTITION_SIZE; i++) {
        partitions.add(i);
    }
    KafkaPublisher publisher = kafkaClient.getPublisher(KafkaPublisher.Ack.FIRE_AND_FORGET, Compression.SNAPPY);
    final KafkaPublisher.Preparer preparer = publisher.prepare(TOPIC_PREFIX);
    // Wait for metrics to be successfully published to Kafka. Retry if publishing fails.
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return publishKafkaMetrics(METRICS_CONTEXT, expected, preparer);
        }
    }, 15, TimeUnit.SECONDS, "Failed to publish correct number of metrics to Kafka");
    // Start KafkaMetricsProcessorService after metrics are published to Kafka
    KafkaMetricsProcessorService kafkaMetricsProcessorService = new KafkaMetricsProcessorService(kafkaClient, injector.getInstance(MetricDatasetFactory.class), new MetricsMessageCallbackFactory(injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, 4), TOPIC_PREFIX, partitions, new NoopMetricsContext());
    kafkaMetricsProcessorService.startAndWait();
    // Intentionally set queue size to a small value, so that MessagingMetricsProcessorService
    // internally can persist metrics when more messages are to be fetched
    MessagingMetricsProcessorService messagingMetricsProcessorService = new MessagingMetricsProcessorService(injector.getInstance(MetricDatasetFactory.class), TOPIC_PREFIX, messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, 1000L, 5, partitions, new NoopMetricsContext(), 50, 0);
    messagingMetricsProcessorService.startAndWait();
    long startTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
    // Publish metrics with messaging service and record expected metrics
    for (int i = 10; i < 20; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.COUNTER);
    }
    Thread.sleep(500);
    // Stop and restart messagingMetricsProcessorService
    messagingMetricsProcessorService.stopAndWait();
    // Intentionally set queue size to a large value, so that MessagingMetricsProcessorService
    // internally only persists metrics during terminating.
    messagingMetricsProcessorService = new MessagingMetricsProcessorService(injector.getInstance(MetricDatasetFactory.class), TOPIC_PREFIX, messagingService, injector.getInstance(SchemaGenerator.class), injector.getInstance(DatumReaderFactory.class), metricStore, 500L, 100, partitions, new NoopMetricsContext(), 50, 0);
    messagingMetricsProcessorService.startAndWait();
    // Publish metrics after MessagingMetricsProcessorService restarts and record expected metrics
    for (int i = 20; i < 30; i++) {
        publishMessagingMetrics(i, startTime, METRICS_CONTEXT, expected, SYSTEM_METRIC_PREFIX, MetricType.GAUGE);
    }
    final List<String> missingMetricNames = new ArrayList<>();
    // are retrieved when timeout occurs, print out the missing metrics
    try {
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return canQueryAllMetrics(metricStore, METRICS_CONTEXT, expected, missingMetricNames);
            }
        }, 10000, TimeUnit.MILLISECONDS, "Failed to get all metrics");
    } catch (TimeoutException e) {
        Assert.fail(String.format("Metrics: [%s] cannot be found in the metrics store.", Joiner.on(", ").join(missingMetricNames)));
    }
    // Query metrics from the metricStore and compare them with the expected ones
    assertMetricsResult(metricStore, METRICS_CONTEXT, expected);
    // Query for the 5 counter metrics published with messaging between time 5 - 14
    Collection<MetricTimeSeries> queryResult = metricStore.query(new MetricDataQuery(5, 14, 1, Integer.MAX_VALUE, ImmutableMap.of(SYSTEM_METRIC_PREFIX + COUNTER_METRIC_NAME, AggregationFunction.SUM), METRICS_CONTEXT, ImmutableList.<String>of(), null));
    MetricTimeSeries timeSeries = Iterables.getOnlyElement(queryResult);
    Assert.assertEquals(5, timeSeries.getTimeValues().size());
    for (TimeValue timeValue : timeSeries.getTimeValues()) {
        Assert.assertEquals(1L, timeValue.getValue());
    }
    // Stop services and servers
    kafkaMetricsProcessorService.stopAndWait();
    messagingMetricsProcessorService.stopAndWait();
    kafkaServer.stopAndWait();
    zkServer.stopAndWait();
    // Delete all metrics
    metricStore.deleteAll();
}