Examples with RetryOnStartFailureService io.cdap.cdap.common.service.RetryOnStartFailureService used on opensource projects

Example 1 with RetryOnStartFailureService

use of io.cdap.cdap.common.service.RetryOnStartFailureService in project cdap by caskdata.

the class AppFabricServiceMain method addServices.

@Override
protected void addServices(Injector injector, List<? super Service> services, List<? super AutoCloseable> closeableResources, MasterEnvironment masterEnv, MasterEnvironmentContext masterEnvContext, EnvironmentOptions options) {
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    if (SecurityUtil.isInternalAuthEnabled(cConf)) {
        services.add(injector.getInstance(TokenManager.class));
    }
    closeableResources.add(injector.getInstance(AccessControllerInstantiator.class));
    services.add(injector.getInstance(OperationalStatsService.class));
    services.add(injector.getInstance(SecureStoreService.class));
    services.add(injector.getInstance(DatasetOpExecutorService.class));
    services.add(injector.getInstance(ServiceStore.class));
    HealthCheckService healthCheckService = injector.getInstance(HealthCheckService.class);
    healthCheckService.helper(Constants.AppFabricHealthCheck.APP_FABRIC_HEALTH_CHECK_SERVICE, cConf, Constants.Service.MASTER_SERVICES_BIND_ADDRESS);
    services.add(healthCheckService);
    Binding<ZKClientService> zkBinding = injector.getExistingBinding(Key.get(ZKClientService.class));
    if (zkBinding != null) {
        services.add(zkBinding.getProvider().get());
    }
    // Start both the remote TwillRunnerService and regular TwillRunnerService
    TwillRunnerService remoteTwillRunner = injector.getInstance(Key.get(TwillRunnerService.class, Constants.AppFabric.RemoteExecution.class));
    services.add(new TwillRunnerServiceWrapper(remoteTwillRunner));
    services.add(new TwillRunnerServiceWrapper(injector.getInstance(TwillRunnerService.class)));
    services.add(new RetryOnStartFailureService(() -> injector.getInstance(DatasetService.class), RetryStrategies.exponentialDelay(200, 5000, TimeUnit.MILLISECONDS)));
    services.add(injector.getInstance(AppFabricServer.class));
    services.add(injector.getInstance(TetheringClientSubscriberService.class));
    if (cConf.getBoolean(Constants.TaskWorker.POOL_ENABLE)) {
        services.add(injector.getInstance(TaskWorkerServiceLauncher.class));
    }
    if (cConf.getBoolean(SystemWorker.POOL_ENABLE)) {
        services.add(injector.getInstance(SystemWorkerServiceLauncher.class));
    }
    // Optionally adds the master environment task
    masterEnv.getTask().ifPresent(task -> services.add(new MasterTaskExecutorService(task, masterEnvContext)));
}

Example 2 with RetryOnStartFailureService

use of io.cdap.cdap.common.service.RetryOnStartFailureService in project cdap by caskdata.

the class LogBufferService method loadLogPipelines.

/**
 * Load log buffer pipelines.
 */
@SuppressWarnings("unchecked")
private List<LogBufferProcessorPipeline> loadLogPipelines() {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    List<LogBufferProcessorPipeline> bufferPipelines = new ArrayList<>();
    // Create one LogBufferProcessorPipeline per spec
    for (LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        CConfiguration cConf = pipelineSpec.getConf();
        AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf);
        LogBufferPipelineConfig config = new LogBufferPipelineConfig(bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS), cConf.getLong(Constants.LogBuffer.LOG_BUFFER_PIPELINE_BATCH_SIZE, 1000));
        CheckpointManager checkpointManager = checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix(), CheckpointManagerFactory.Type.LOG_BUFFER);
        LogBufferProcessorPipeline pipeline = new LogBufferProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), config, checkpointManager, 0);
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> pipeline, retryStrategy));
        bufferPipelines.add(pipeline);
        checkpointManagers.add(checkpointManager);
    }
    return bufferPipelines;
}

Example 3 with RetryOnStartFailureService

use of io.cdap.cdap.common.service.RetryOnStartFailureService in project cdap by caskdata.

the class DistributedLogFramework method createService.

@Override
@SuppressWarnings("unchecked")
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix() + topic, CheckpointManagerFactory.Type.KAFKA), brokerService, config), retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::start));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::stop));
        }
    };
}