Search in sources :

Example 1 with ServiceLoggingContext

use of io.cdap.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class PreviewRunnerTwillRunnable method doInitialize.

private void doInitialize(TwillContext context) throws Exception {
    CConfiguration cConf = CConfiguration.create(new File(getArgument("cConf")).toURI().toURL());
    Configuration hConf = new Configuration();
    hConf.clear();
    hConf.addResource(new File(getArgument("hConf")).toURI().toURL());
    PreviewRequestPollerInfo pollerInfo;
    if (context instanceof ExtendedTwillContext) {
        pollerInfo = new PreviewRequestPollerInfo(context.getInstanceId(), ((ExtendedTwillContext) context).getUID());
    } else {
        pollerInfo = new PreviewRequestPollerInfo(context.getInstanceId(), null);
    }
    LOG.debug("Initializing preview runner with poller info {} in total {} runners", pollerInfo, context.getInstanceCount());
    Injector injector = createInjector(cConf, hConf, pollerInfo);
    // Initialize logging context
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    LoggingContext loggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, PreviewRunnerTwillApplication.NAME);
    LoggingContextAccessor.setLoggingContext(loggingContext);
    // Optionally get the storage provider. It is for destroy() method to close it on shutdown.
    Binding<StorageProvider> storageBinding = injector.getExistingBinding(Key.get(StorageProvider.class));
    if (storageBinding != null) {
        storageProvider = storageBinding.getProvider().get();
    }
    previewRunnerManager = injector.getInstance(PreviewRunnerManager.class);
}
Also used : ExtendedTwillContext(io.cdap.cdap.master.spi.twill.ExtendedTwillContext) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) Configuration(org.apache.hadoop.conf.Configuration) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) Injector(com.google.inject.Injector) PreviewRunnerManager(io.cdap.cdap.app.preview.PreviewRunnerManager) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) StorageProvider(io.cdap.cdap.spi.data.StorageProvider) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) File(java.io.File) PreviewRequestPollerInfo(io.cdap.cdap.internal.app.runtime.k8s.PreviewRequestPollerInfo)

Example 2 with ServiceLoggingContext

use of io.cdap.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class SystemWorkerTwillRunnable method doInitialize.

private void doInitialize(TwillContext context) throws Exception {
    CConfiguration cConf = CConfiguration.create(new File(getArgument("cConf")).toURI().toURL());
    // Overwrite the app fabric temp directory with the task worker temp directory
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, cConf.get(Constants.TaskWorker.LOCAL_DATA_DIR));
    Configuration hConf = new Configuration();
    hConf.clear();
    hConf.addResource(new File(getArgument("hConf")).toURI().toURL());
    Injector injector = createInjector(cConf, hConf);
    // Initialize logging context
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    LoggingContext loggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, SystemWorkerTwillApplication.NAME);
    LoggingContextAccessor.setLoggingContext(loggingContext);
    taskWorker = injector.getInstance(TaskWorkerService.class);
}
Also used : LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) Injector(com.google.inject.Injector) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) File(java.io.File) TaskWorkerService(io.cdap.cdap.internal.app.worker.TaskWorkerService)

Example 3 with ServiceLoggingContext

use of io.cdap.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class DefaultPreviewManager method startUp.

@Override
protected void startUp() throws Exception {
    previewInjector = createPreviewInjector();
    StoreDefinition.createAllTables(previewInjector.getInstance(StructuredTableAdmin.class));
    metricsCollectionService.start();
    logAppender = previewInjector.getInstance(LogAppender.class);
    logAppender.start();
    LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP));
    logSubscriberService = previewInjector.getInstance(PreviewTMSLogSubscriber.class);
    logSubscriberService.startAndWait();
    dataSubscriberService = previewInjector.getInstance(PreviewDataSubscriberService.class);
    dataSubscriberService.startAndWait();
    previewDataCleanupService.startAndWait();
}
Also used : StructuredTableAdmin(io.cdap.cdap.spi.data.StructuredTableAdmin) LogAppender(io.cdap.cdap.logging.appender.LogAppender) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext)

Example 4 with ServiceLoggingContext

use of io.cdap.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class DefaultPreviewRunner method startUp.

@Override
protected void startUp() throws Exception {
    LOG.debug("Starting preview runner service");
    StoreDefinition.createAllTables(structuredTableAdmin);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    dsOpExecService.startAndWait();
    datasetService.startAndWait();
    // It is recommended to initialize log appender after datasetService is started,
    // since log appender instantiates a dataset.
    logAppenderInitializer.initialize();
    LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP));
    Futures.allAsList(applicationLifecycleService.start(), programRuntimeService.start(), metricsCollectionService.start(), programNotificationSubscriberService.start()).get();
    Files.createDirectories(previewIdDirPath);
    // Reconcile status for abruptly terminated preview runs
    try (Stream<Path> paths = Files.walk(Paths.get(previewIdDirPath.toString()))) {
        paths.filter(Files::isRegularFile).forEach(path -> {
            try (Reader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
                ProgramId programId = GSON.fromJson(reader, ProgramId.class);
                long submitTimeMillis = RunIds.getTime(programId.getApplication(), TimeUnit.MILLISECONDS);
                PreviewStatus status = new PreviewStatus(PreviewStatus.Status.KILLED_BY_EXCEEDING_MEMORY_LIMIT, submitTimeMillis, new BasicThrowable(new Exception("Preview runner container killed possibly because of out of memory. " + "Please try running preview again.")), null, null);
                previewTerminated(programId, status);
            } catch (IOException e) {
                LOG.warn("Error reading file {}. Ignoring", path, e);
            }
        });
    }
}
Also used : Path(java.nio.file.Path) PreviewStatus(io.cdap.cdap.app.preview.PreviewStatus) ProgramNotificationSubscriberService(io.cdap.cdap.internal.app.services.ProgramNotificationSubscriberService) MessagingService(io.cdap.cdap.messaging.MessagingService) ApplicationLifecycleService(io.cdap.cdap.internal.app.services.ApplicationLifecycleService) DatasetService(io.cdap.cdap.data2.datafabric.dataset.service.DatasetService) LevelDBTableService(io.cdap.cdap.data2.dataset2.lib.table.leveldb.LevelDBTableService) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) DatasetOpExecutorService(io.cdap.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutorService) ProgramLifecycleService(io.cdap.cdap.internal.app.services.ProgramLifecycleService) ProgramRuntimeService(io.cdap.cdap.app.runtime.ProgramRuntimeService) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) Reader(java.io.Reader) IOException(java.io.IOException) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) ProgramId(io.cdap.cdap.proto.id.ProgramId) BasicThrowable(io.cdap.cdap.proto.BasicThrowable) TimeoutException(java.util.concurrent.TimeoutException) NamespaceAlreadyExistsException(io.cdap.cdap.common.NamespaceAlreadyExistsException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException)

Example 5 with ServiceLoggingContext

use of io.cdap.cdap.common.logging.ServiceLoggingContext in project cdap by caskdata.

the class PreviewHttpServer method startUp.

/**
 * Configures the AppFabricService pre-start.
 */
@Override
protected void startUp() throws Exception {
    LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.PREVIEW_HTTP));
    if (previewManager instanceof Service) {
        ((Service) previewManager).startAndWait();
    }
    httpService.start();
    cancelHttpService = discoveryService.register(ResolvingDiscoverable.of(URIScheme.createDiscoverable(Constants.Service.PREVIEW_HTTP, httpService)));
    LOG.info("Preview HTTP server started on {}", httpService.getBindAddress());
}
Also used : AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) NettyHttpService(io.cdap.http.NettyHttpService) DiscoveryService(org.apache.twill.discovery.DiscoveryService) Service(com.google.common.util.concurrent.Service) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext)

Aggregations

ServiceLoggingContext (io.cdap.cdap.common.logging.ServiceLoggingContext)23 LogAppenderInitializer (io.cdap.cdap.logging.appender.LogAppenderInitializer)10 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)9 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)6 LoggingContext (io.cdap.cdap.common.logging.LoggingContext)6 Configuration (org.apache.hadoop.conf.Configuration)6 Injector (com.google.inject.Injector)5 File (java.io.File)5 AbstractIdleService (com.google.common.util.concurrent.AbstractIdleService)3 ImmutableList (com.google.common.collect.ImmutableList)2 Service (com.google.common.util.concurrent.Service)2 ProgramRuntimeService (io.cdap.cdap.app.runtime.ProgramRuntimeService)2 SConfiguration (io.cdap.cdap.common.conf.SConfiguration)2 LoggerContext (ch.qos.logback.classic.LoggerContext)1 Futures (com.google.common.util.concurrent.Futures)1 Inject (com.google.inject.Inject)1 Named (com.google.inject.name.Named)1 MetricDataQuery (io.cdap.cdap.api.metrics.MetricDataQuery)1 MetricStore (io.cdap.cdap.api.metrics.MetricStore)1 PreviewRunnerManager (io.cdap.cdap.app.preview.PreviewRunnerManager)1