Search in sources :

Example 1 with KafkaPipelineConfig

use of io.cdap.cdap.logging.pipeline.kafka.KafkaPipelineConfig in project cdap by caskdata.

the class DistributedLogFramework method createService.

@Override
@SuppressWarnings("unchecked")
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix() + topic, CheckpointManagerFactory.Type.KAFKA), brokerService, config), retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::start));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::stop));
        }
    };
}
Also used : LogPipelineSpecification(io.cdap.cdap.logging.framework.LogPipelineSpecification) ArrayList(java.util.ArrayList) KafkaPipelineConfig(io.cdap.cdap.logging.pipeline.kafka.KafkaPipelineConfig) RetryOnStartFailureService(io.cdap.cdap.common.service.RetryOnStartFailureService) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) DiscoveryService(org.apache.twill.discovery.DiscoveryService) ResourceBalancerService(io.cdap.cdap.common.resource.ResourceBalancerService) Service(com.google.common.util.concurrent.Service) BrokerService(org.apache.twill.kafka.client.BrokerService) LogPipelineLoader(io.cdap.cdap.logging.framework.LogPipelineLoader) LogProcessorPipelineContext(io.cdap.cdap.logging.pipeline.LogProcessorPipelineContext) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) KafkaLogProcessorPipeline(io.cdap.cdap.logging.pipeline.kafka.KafkaLogProcessorPipeline) AppenderContext(io.cdap.cdap.api.logging.AppenderContext) RetryOnStartFailureService(io.cdap.cdap.common.service.RetryOnStartFailureService) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) RetryStrategy(io.cdap.cdap.common.service.RetryStrategy)

Aggregations

AbstractIdleService (com.google.common.util.concurrent.AbstractIdleService)1 Service (com.google.common.util.concurrent.Service)1 AppenderContext (io.cdap.cdap.api.logging.AppenderContext)1 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)1 ResourceBalancerService (io.cdap.cdap.common.resource.ResourceBalancerService)1 RetryOnStartFailureService (io.cdap.cdap.common.service.RetryOnStartFailureService)1 RetryStrategy (io.cdap.cdap.common.service.RetryStrategy)1 LogPipelineLoader (io.cdap.cdap.logging.framework.LogPipelineLoader)1 LogPipelineSpecification (io.cdap.cdap.logging.framework.LogPipelineSpecification)1 LogProcessorPipelineContext (io.cdap.cdap.logging.pipeline.LogProcessorPipelineContext)1 KafkaLogProcessorPipeline (io.cdap.cdap.logging.pipeline.kafka.KafkaLogProcessorPipeline)1 KafkaPipelineConfig (io.cdap.cdap.logging.pipeline.kafka.KafkaPipelineConfig)1 ArrayList (java.util.ArrayList)1 DiscoveryService (org.apache.twill.discovery.DiscoveryService)1 BrokerService (org.apache.twill.kafka.client.BrokerService)1