Examples with LogAppenderInitializer io.cdap.cdap.logging.appender.LogAppenderInitializer used on opensource projects

Example 6 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by cdapio.

the class TestTMSLogging method testTmsLogAppender.

@Test
public void testTmsLogAppender() throws Exception {
    // setup TMSLogAppender and log messages to it
    LogAppenderInitializer logAppenderInitializer = new LogAppenderInitializer(tmsLogAppender);
    logAppenderInitializer.initialize("TestTMSLogging");
    Logger logger = LoggerFactory.getLogger("TestTMSLogging");
    LoggingTester loggingTester = new LoggingTester();
    LoggingContext loggingContext = new MapReduceLoggingContext("TKL_NS_1", "APP_1", "MR_1", "RUN1");
    loggingTester.generateLogs(logger, loggingContext);
    logAppenderInitializer.close();
    // fetch and deserialize all the logs from TMS
    LoggingEventSerializer loggingEventSerializer = new LoggingEventSerializer();
    Map<Integer, List<ILoggingEvent>> partitionedFetchedLogs = new HashMap<>();
    int totalFetchedLogs = 0;
    for (Map.Entry<Integer, TopicId> topicId : topicIds.entrySet()) {
        List<ILoggingEvent> fetchedLogs = new ArrayList<>();
        MessageFetcher messageFetcher = client.prepareFetch(topicId.getValue());
        try (CloseableIterator<RawMessage> messages = messageFetcher.fetch()) {
            while (messages.hasNext()) {
                RawMessage message = messages.next();
                ILoggingEvent iLoggingEvent = loggingEventSerializer.fromBytes(ByteBuffer.wrap(message.getPayload()));
                fetchedLogs.add(iLoggingEvent);
            }
        }
        totalFetchedLogs += fetchedLogs.size();
        partitionedFetchedLogs.put(topicId.getKey(), fetchedLogs);
    }
    // LoggingTester emits 240 logs in total
    Assert.assertEquals(240, totalFetchedLogs);
    // Read the partition that our LoggingContext maps to and filter the logs in there to the logs that correspond
    // to our LoggingContext.
    LogPartitionType logPartitionType = LogPartitionType.valueOf(cConf.get(Constants.Logging.LOG_PUBLISH_PARTITION_KEY).toUpperCase());
    String partitionKey = logPartitionType.getPartitionKey(loggingContext);
    int partition = TMSLogAppender.partition(partitionKey, cConf.getInt(Constants.Logging.NUM_PARTITIONS));
    Filter logFilter = LoggingContextHelper.createFilter(loggingContext);
    List<ILoggingEvent> filteredLogs = partitionedFetchedLogs.get(partition).stream().filter(logFilter::match).collect(Collectors.toList());
    // LoggingTester emits 60 logs with the given LoggingContext
    Assert.assertEquals(60, filteredLogs.size());
    for (int i = 0; i < filteredLogs.size(); i++) {
        ILoggingEvent loggingEvent = filteredLogs.get(i);
        Assert.assertEquals(String.format("Test log message %s arg1 arg2", i), loggingEvent.getFormattedMessage());
    }
}

Example 7 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by cdapio.

the class TestFileLogging method setUpContext.

@BeforeClass
public static void setUpContext() throws Exception {
    Configuration hConf = HBaseConfiguration.create();
    final CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    String logBaseDir = cConf.get(LoggingConfiguration.LOG_BASE_DIR) + "/" + TestFileLogging.class.getSimpleName();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    injector = Guice.createInjector(new ConfigModule(cConf, hConf), new NonCustomLocationUnitTestModule(), new TransactionModules().getInMemoryModules(), new LocalLogAppenderModule(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new StorageModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
        }
    });
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    StoreDefinition.LogFileMetaStore.create(injector.getInstance(StructuredTableAdmin.class));
    LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestFileLogging");
    Logger logger = LoggerFactory.getLogger("TestFileLogging");
    LoggingTester loggingTester = new LoggingTester();
    loggingTester.generateLogs(logger, new WorkerLoggingContext("TFL_NS_1", "APP_1", "WORKER_1", "RUN1", "INSTANCE1"));
    appender.stop();
}

Example 8 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by cdapio.

the class TestKafkaLogging method init.

@BeforeClass
public static void init() throws Exception {
    KafkaLogAppender appender = KAFKA_TESTER.getInjector().getInstance(KafkaLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestKafkaLogging");
    Logger logger = LoggerFactory.getLogger("TestKafkaLogging");
    LoggingTester loggingTester = new LoggingTester();
    loggingTester.generateLogs(logger, new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1"));
    appender.stop();
}

Example 9 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by cdapio.

the class SparkRuntimeContextProvider method createIfNotExists.

/**
 * Creates a singleton {@link SparkRuntimeContext}.
 * It has assumption on file location that are localized by the SparkRuntimeService.
 */
private static synchronized SparkRuntimeContext createIfNotExists() {
    if (sparkRuntimeContext != null) {
        return sparkRuntimeContext;
    }
    try {
        CConfiguration cConf = createCConf();
        Configuration hConf = createHConf();
        SparkRuntimeContextConfig contextConfig = new SparkRuntimeContextConfig(hConf);
        ProgramOptions programOptions = contextConfig.getProgramOptions();
        ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
        if (masterEnvName != null) {
            MasterEnvironment masterEnv = MasterEnvironments.create(cConf, masterEnvName);
            MasterEnvironmentContext context = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
            masterEnv.initialize(context);
            MasterEnvironments.setMasterEnvironment(masterEnv);
        }
        // Create the program
        Program program = createProgram(cConf, contextConfig);
        ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(programOptions));
        Injector injector = createInjector(cConf, hConf, contextConfig.getProgramId(), programOptions);
        LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
        logAppenderInitializer.initialize();
        SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
        ProxySelector oldProxySelector = ProxySelector.getDefault();
        if (clusterMode == ClusterMode.ISOLATED) {
            RuntimeMonitors.setupMonitoring(injector, programOptions);
        }
        MetricsCollectionService metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
        SparkServiceAnnouncer serviceAnnouncer = injector.getInstance(SparkServiceAnnouncer.class);
        Deque<Service> coreServices = new LinkedList<>();
        if (clusterMode == ClusterMode.ON_PREMISE && masterEnvName == null) {
            // Add ZK for discovery and Kafka
            coreServices.add(injector.getInstance(ZKClientService.class));
            // Add the Kafka client for logs collection
            coreServices.add(injector.getInstance(KafkaClientService.class));
        }
        coreServices.add(metricsCollectionService);
        coreServices.add(serviceAnnouncer);
        for (Service coreService : coreServices) {
            coreService.startAndWait();
        }
        AtomicBoolean closed = new AtomicBoolean();
        Closeable closeable = () -> {
            if (!closed.compareAndSet(false, true)) {
                return;
            }
            // Close to flush out all important logs
            logAppenderInitializer.close();
            // Stop all services. Reverse the order.
            for (Service service : (Iterable<Service>) coreServices::descendingIterator) {
                try {
                    service.stopAndWait();
                } catch (Exception e) {
                    LOG.warn("Exception raised when stopping service {} during program termination.", service, e);
                }
            }
            Authenticator.setDefault(null);
            ProxySelector.setDefault(oldProxySelector);
            LOG.debug("Spark runtime services shutdown completed");
        };
        // Constructor the DatasetFramework
        DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
        WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, contextConfig.getApplicationSpecification());
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(cConf, contextConfig, program.getClassLoader());
        // Create the context object
        sparkRuntimeContext = new SparkRuntimeContext(contextConfig.getConfiguration(), program, programOptions, cConf, getHostname(), injector.getInstance(TransactionSystemClient.class), programDatasetFramework, metricsCollectionService, contextConfig.getWorkflowProgramInfo(), pluginInstantiator, injector.getInstance(SecureStore.class), injector.getInstance(SecureStoreManager.class), injector.getInstance(AccessEnforcer.class), injector.getInstance(AuthenticationContext.class), injector.getInstance(MessagingService.class), serviceAnnouncer, injector.getInstance(PluginFinder.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetadataReader.class), injector.getInstance(MetadataPublisher.class), injector.getInstance(NamespaceQueryAdmin.class), injector.getInstance(FieldLineageWriter.class), injector.getInstance(RemoteClientFactory.class), closeable);
        LoggingContextAccessor.setLoggingContext(sparkRuntimeContext.getLoggingContext());
        return sparkRuntimeContext;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 10 with LogAppenderInitializer

use of io.cdap.cdap.logging.appender.LogAppenderInitializer in project cdap by cdapio.

the class SystemWorkerTwillRunnable method doInitialize.

private void doInitialize(TwillContext context) throws Exception {
    CConfiguration cConf = CConfiguration.create(new File(getArgument("cConf")).toURI().toURL());
    // Overwrite the app fabric temp directory with the task worker temp directory
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, cConf.get(Constants.TaskWorker.LOCAL_DATA_DIR));
    Configuration hConf = new Configuration();
    hConf.clear();
    hConf.addResource(new File(getArgument("hConf")).toURI().toURL());
    Injector injector = createInjector(cConf, hConf);
    // Initialize logging context
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    LoggingContext loggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, SystemWorkerTwillApplication.NAME);
    LoggingContextAccessor.setLoggingContext(loggingContext);
    taskWorker = injector.getInstance(TaskWorkerService.class);
}