Search in sources :

Example 1 with KafkaPipelineConfig

use of co.cask.cdap.logging.pipeline.kafka.KafkaPipelineConfig in project cdap by caskdata.

the class DistributedLogFramework method createService.

@Override
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(new Supplier<Service>() {

            @Override
            public Service get() {
                return new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(topic, pipelineSpec.getCheckpointPrefix()), brokerService, config);
            }
        }, retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, new Function<Service, ListenableFuture<State>>() {

                @Override
                public ListenableFuture<State> apply(Service service) {
                    return service.start();
                }
            }));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, new Function<Service, ListenableFuture<State>>() {

                @Override
                public ListenableFuture<State> apply(Service service) {
                    return service.stop();
                }
            }));
        }
    };
}
Also used : LogPipelineSpecification(co.cask.cdap.logging.framework.LogPipelineSpecification) ArrayList(java.util.ArrayList) KafkaPipelineConfig(co.cask.cdap.logging.pipeline.kafka.KafkaPipelineConfig) ResourceBalancerService(co.cask.cdap.common.resource.ResourceBalancerService) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) RetryOnStartFailureService(co.cask.cdap.common.service.RetryOnStartFailureService) DiscoveryService(org.apache.twill.discovery.DiscoveryService) Service(com.google.common.util.concurrent.Service) BrokerService(org.apache.twill.kafka.client.BrokerService) LogPipelineLoader(co.cask.cdap.logging.framework.LogPipelineLoader) LogProcessorPipelineContext(co.cask.cdap.logging.pipeline.LogProcessorPipelineContext) CConfiguration(co.cask.cdap.common.conf.CConfiguration) Function(com.google.common.base.Function) KafkaLogProcessorPipeline(co.cask.cdap.logging.pipeline.kafka.KafkaLogProcessorPipeline) AppenderContext(co.cask.cdap.api.logging.AppenderContext) RetryOnStartFailureService(co.cask.cdap.common.service.RetryOnStartFailureService) Supplier(com.google.common.base.Supplier) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) RetryStrategy(co.cask.cdap.common.service.RetryStrategy)

Aggregations

AppenderContext (co.cask.cdap.api.logging.AppenderContext)1 CConfiguration (co.cask.cdap.common.conf.CConfiguration)1 ResourceBalancerService (co.cask.cdap.common.resource.ResourceBalancerService)1 RetryOnStartFailureService (co.cask.cdap.common.service.RetryOnStartFailureService)1 RetryStrategy (co.cask.cdap.common.service.RetryStrategy)1 LogPipelineLoader (co.cask.cdap.logging.framework.LogPipelineLoader)1 LogPipelineSpecification (co.cask.cdap.logging.framework.LogPipelineSpecification)1 LogProcessorPipelineContext (co.cask.cdap.logging.pipeline.LogProcessorPipelineContext)1 KafkaLogProcessorPipeline (co.cask.cdap.logging.pipeline.kafka.KafkaLogProcessorPipeline)1 KafkaPipelineConfig (co.cask.cdap.logging.pipeline.kafka.KafkaPipelineConfig)1 Function (com.google.common.base.Function)1 Supplier (com.google.common.base.Supplier)1 AbstractIdleService (com.google.common.util.concurrent.AbstractIdleService)1 Service (com.google.common.util.concurrent.Service)1 ArrayList (java.util.ArrayList)1 DiscoveryService (org.apache.twill.discovery.DiscoveryService)1 BrokerService (org.apache.twill.kafka.client.BrokerService)1