Search in sources :

Example 1 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by caskdata.

the class AbstractProgramTwillRunnable method doInitialize.

/**
 * Prepares this instance to execute a program.
 *
 * @param programOptionFile a json file containing the serialized {@link ProgramOptions}
 * @throws Exception if failed to initialize
 */
private void doInitialize(File programOptionFile) throws Exception {
    controllerFuture = new CompletableFuture<>();
    programCompletion = new CompletableFuture<>();
    // Setup process wide settings
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    System.setSecurityManager(new ProgramContainerSecurityManager(System.getSecurityManager()));
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();
    // Create the ProgramOptions
    programOptions = createProgramOptions(programOptionFile);
    programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions));
    Arguments systemArgs = programOptions.getArguments();
    LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
    ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
    // Loads configurations
    Configuration hConf = new Configuration();
    if (clusterMode == ClusterMode.ON_PREMISE) {
        hConf.clear();
        hConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.HADOOP_CONF_FILE)).toURI().toURL());
    }
    UserGroupInformation.setConfiguration(hConf);
    CConfiguration cConf = CConfiguration.create();
    cConf.clear();
    cConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.CDAP_CONF_FILE)).toURI().toURL());
    maxStopSeconds = cConf.getLong(io.cdap.cdap.common.conf.Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS);
    Injector injector = Guice.createInjector(createModule(cConf, hConf, programOptions, programRunId));
    // Initialize log appender
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
    // Setup the proxy selector for in active monitoring mode
    oldProxySelector = ProxySelector.getDefault();
    if (clusterMode == ClusterMode.ISOLATED) {
        RuntimeMonitors.setupMonitoring(injector, programOptions);
    }
    // Create list of core services. They'll will be started in the run method and shutdown when the run
    // method completed
    coreServices = createCoreServices(injector, programOptions);
    // Create the ProgramRunner
    programRunner = createProgramRunner(injector);
    // Create the Program instance
    Location programJarLocation = Locations.toLocation(new File(systemArgs.getOption(ProgramOptionConstants.PROGRAM_JAR)));
    ApplicationSpecification appSpec = readJsonFile(new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class);
    // Expand the program jar for creating classloader
    ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> new File("expanded." + System.currentTimeMillis() + programJarLocation.getName()));
    program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, classLoaderFolder.getDir());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClusterMode(io.cdap.cdap.app.guice.ClusterMode) Arguments(io.cdap.cdap.app.runtime.Arguments) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ClassLoaderFolder(io.cdap.cdap.common.lang.jar.ClassLoaderFolder) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) Injector(com.google.inject.Injector) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) UncaughtExceptionHandler(io.cdap.cdap.common.logging.common.UncaughtExceptionHandler) File(java.io.File) Location(org.apache.twill.filesystem.Location)

Example 2 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by caskdata.

the class PreviewRunnerTwillRunnable method doInitialize.

private void doInitialize(TwillContext context) throws Exception {
    CConfiguration cConf = CConfiguration.create(new File(getArgument("cConf")).toURI().toURL());
    Configuration hConf = new Configuration();
    hConf.clear();
    hConf.addResource(new File(getArgument("hConf")).toURI().toURL());
    PreviewRequestPollerInfo pollerInfo;
    if (context instanceof ExtendedTwillContext) {
        pollerInfo = new PreviewRequestPollerInfo(context.getInstanceId(), ((ExtendedTwillContext) context).getUID());
    } else {
        pollerInfo = new PreviewRequestPollerInfo(context.getInstanceId(), null);
    }
    LOG.debug("Initializing preview runner with poller info {} in total {} runners", pollerInfo, context.getInstanceCount());
    Injector injector = createInjector(cConf, hConf, pollerInfo);
    // Initialize logging context
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    LoggingContext loggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, PreviewRunnerTwillApplication.NAME);
    LoggingContextAccessor.setLoggingContext(loggingContext);
    // Optionally get the storage provider. It is for destroy() method to close it on shutdown.
    Binding<StorageProvider> storageBinding = injector.getExistingBinding(Key.get(StorageProvider.class));
    if (storageBinding != null) {
        storageProvider = storageBinding.getProvider().get();
    }
    previewRunnerManager = injector.getInstance(PreviewRunnerManager.class);
}
Also used : ExtendedTwillContext(io.cdap.cdap.master.spi.twill.ExtendedTwillContext) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) Configuration(org.apache.hadoop.conf.Configuration) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) Injector(com.google.inject.Injector) PreviewRunnerManager(io.cdap.cdap.app.preview.PreviewRunnerManager) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) StorageProvider(io.cdap.cdap.spi.data.StorageProvider) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) File(java.io.File) PreviewRequestPollerInfo(io.cdap.cdap.internal.app.runtime.k8s.PreviewRequestPollerInfo)

Example 3 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by caskdata.

the class AbstractServiceMain method init.

@Override
public final void init(String[] args) throws Exception {
    LOG.info("Initializing master service class {}", getClass().getName());
    // System wide setup
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    // Intercept JUL loggers
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();
    TypeToken<?> type = TypeToken.of(getClass()).resolveType(AbstractServiceMain.class.getTypeParameters()[0]);
    T options = (T) type.getRawType().newInstance();
    OptionsParser.init(options, args, getClass().getSimpleName(), ProjectInfo.getVersion().toString(), System.out);
    CConfiguration cConf = CConfiguration.create();
    SecurityUtil.loginForMasterService(cConf);
    SConfiguration sConf = SConfiguration.create();
    if (options.getExtraConfPath() != null) {
        cConf.addResource(new File(options.getExtraConfPath(), "cdap-site.xml").toURI().toURL());
        sConf.addResource(new File(options.getExtraConfPath(), "cdap-security.xml").toURI().toURL());
    }
    cConf = updateCConf(cConf);
    Configuration hConf = new Configuration();
    masterEnv = MasterEnvironments.setMasterEnvironment(MasterEnvironments.create(cConf, options.getEnvProvider()));
    MasterEnvironmentContext masterEnvContext = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
    masterEnv.initialize(masterEnvContext);
    List<Module> modules = new ArrayList<>();
    modules.add(new ConfigModule(cConf, hConf, sConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new PreviewConfigModule(cConf, hConf, sConf));
    modules.add(new IOModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
            bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
        }
    });
    modules.add(getLogAppenderModule());
    CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
    modules.add(coreSecurityModule);
    if (coreSecurityModule.requiresZKClient()) {
        modules.add(new ZKClientModule());
    }
    modules.add(new AuthenticationContextModules().getMasterModule());
    modules.addAll(getServiceModules(masterEnv, options, cConf));
    injector = Guice.createInjector(modules);
    // Initialize logging context
    LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    closeableResources.add(logAppenderInitializer);
    logAppenderInitializer.initialize();
    Optional.ofNullable(getLoggingContext(options)).ifPresent(LoggingContextAccessor::setLoggingContext);
    // Add Services
    services.add(injector.getInstance(MetricsCollectionService.class));
    addServices(injector, services, closeableResources, masterEnv, masterEnvContext, options);
    // Optionally get the storage provider. It is for destroy() method to close it on shutdown.
    Binding<StorageProvider> storageBinding = injector.getExistingBinding(Key.get(StorageProvider.class));
    if (storageBinding != null) {
        storageProvider = storageBinding.getProvider().get();
    }
    LOG.info("Service {} initialized", getClass().getName());
}
Also used : IOModule(io.cdap.cdap.common.guice.IOModule) Configuration(org.apache.hadoop.conf.Configuration) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) PreviewConfigModule(io.cdap.cdap.app.preview.PreviewConfigModule) ArrayList(java.util.ArrayList) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) ZKClientModule(io.cdap.cdap.common.guice.ZKClientModule) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) PreviewConfigModule(io.cdap.cdap.app.preview.PreviewConfigModule) CoreSecurityModule(io.cdap.cdap.security.guice.CoreSecurityModule) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) SupplierProviderBridge(io.cdap.cdap.common.guice.SupplierProviderBridge) UncaughtExceptionHandler(io.cdap.cdap.common.logging.common.UncaughtExceptionHandler) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) AuthenticationContextModules(io.cdap.cdap.security.auth.context.AuthenticationContextModules) StorageProvider(io.cdap.cdap.spi.data.StorageProvider) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) AbstractModule(com.google.inject.AbstractModule) MasterEnvironmentContext(io.cdap.cdap.master.spi.environment.MasterEnvironmentContext) Module(com.google.inject.Module) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) PreviewConfigModule(io.cdap.cdap.app.preview.PreviewConfigModule) CoreSecurityModule(io.cdap.cdap.security.guice.CoreSecurityModule) TransactionExecutorModule(io.cdap.cdap.data.runtime.TransactionExecutorModule) RemoteLogAppenderModule(io.cdap.cdap.logging.guice.RemoteLogAppenderModule) CoreSecurityRuntimeModule(io.cdap.cdap.security.guice.CoreSecurityRuntimeModule) ZKClientModule(io.cdap.cdap.common.guice.ZKClientModule) StorageModule(io.cdap.cdap.data.runtime.StorageModule) IOModule(io.cdap.cdap.common.guice.IOModule) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) AbstractModule(com.google.inject.AbstractModule) LoggingContextAccessor(io.cdap.cdap.common.logging.LoggingContextAccessor) File(java.io.File)

Example 4 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by caskdata.

the class SystemWorkerTwillRunnable method doInitialize.

private void doInitialize(TwillContext context) throws Exception {
    CConfiguration cConf = CConfiguration.create(new File(getArgument("cConf")).toURI().toURL());
    // Overwrite the app fabric temp directory with the task worker temp directory
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, cConf.get(Constants.TaskWorker.LOCAL_DATA_DIR));
    Configuration hConf = new Configuration();
    hConf.clear();
    hConf.addResource(new File(getArgument("hConf")).toURI().toURL());
    Injector injector = createInjector(cConf, hConf);
    // Initialize logging context
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    LoggingContext loggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, SystemWorkerTwillApplication.NAME);
    LoggingContextAccessor.setLoggingContext(loggingContext);
    taskWorker = injector.getInstance(TaskWorkerService.class);
}
Also used : LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) Injector(com.google.inject.Injector) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) File(java.io.File) TaskWorkerService(io.cdap.cdap.internal.app.worker.TaskWorkerService)

Example 5 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by caskdata.

the class TestKafkaLogging method init.

@BeforeClass
public static void init() throws Exception {
    KafkaLogAppender appender = KAFKA_TESTER.getInjector().getInstance(KafkaLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestKafkaLogging");
    Logger logger = LoggerFactory.getLogger("TestKafkaLogging");
    LoggingTester loggingTester = new LoggingTester();
    loggingTester.generateLogs(logger, new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1"));
    appender.stop();
}
Also used : LoggingTester(io.cdap.cdap.logging.appender.LoggingTester) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) Logger(org.slf4j.Logger) BeforeClass(org.junit.BeforeClass)

Aggregations

LogAppenderInitializer (io.cdap.cdap.logging.appender.LogAppenderInitializer)11 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)9 Configuration (org.apache.hadoop.conf.Configuration)8 Injector (com.google.inject.Injector)7 File (java.io.File)7 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)6 LoggingContext (io.cdap.cdap.common.logging.LoggingContext)5 ServiceLoggingContext (io.cdap.cdap.common.logging.ServiceLoggingContext)4 UncaughtExceptionHandler (io.cdap.cdap.common.logging.common.UncaughtExceptionHandler)3 LoggingTester (io.cdap.cdap.logging.appender.LoggingTester)3 AbstractIdleService (com.google.common.util.concurrent.AbstractIdleService)2 Service (com.google.common.util.concurrent.Service)2 AbstractModule (com.google.inject.AbstractModule)2 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)2 ClusterMode (io.cdap.cdap.app.guice.ClusterMode)2 Program (io.cdap.cdap.app.program.Program)2 ProgramDescriptor (io.cdap.cdap.app.program.ProgramDescriptor)2 Arguments (io.cdap.cdap.app.runtime.Arguments)2 ProgramOptions (io.cdap.cdap.app.runtime.ProgramOptions)2 SConfiguration (io.cdap.cdap.common.conf.SConfiguration)2