Search in sources :

Example 1 with WorkerLoggingContext

use of io.cdap.cdap.logging.context.WorkerLoggingContext in project cdap by caskdata.

the class TestFileLogging method testGetLogNext.

@Test
public void testGetLogNext() throws Exception {
    LoggingContext loggingContext = new WorkerLoggingContext("TFL_NS_1", "APP_1", "WORKER_1", "RUN1", "INSTANCE1");
    FileLogReader logReader = injector.getInstance(FileLogReader.class);
    LoggingTester tester = new LoggingTester();
    tester.testGetNext(logReader, loggingContext);
}
Also used : LoggingTester(io.cdap.cdap.logging.appender.LoggingTester) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) FileLogReader(io.cdap.cdap.logging.read.FileLogReader) Test(org.junit.Test)

Example 2 with WorkerLoggingContext

use of io.cdap.cdap.logging.context.WorkerLoggingContext in project cdap by caskdata.

the class TestKafkaLogging method init.

@BeforeClass
public static void init() throws Exception {
    KafkaLogAppender appender = KAFKA_TESTER.getInjector().getInstance(KafkaLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestKafkaLogging");
    Logger logger = LoggerFactory.getLogger("TestKafkaLogging");
    LoggingTester loggingTester = new LoggingTester();
    loggingTester.generateLogs(logger, new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1"));
    appender.stop();
}
Also used : LoggingTester(io.cdap.cdap.logging.appender.LoggingTester) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) Logger(org.slf4j.Logger) BeforeClass(org.junit.BeforeClass)

Example 3 with WorkerLoggingContext

use of io.cdap.cdap.logging.context.WorkerLoggingContext in project cdap by caskdata.

the class TestKafkaLogging method testGetNext.

@Test
public void testGetNext() throws Exception {
    // Check with null runId and null instanceId
    LoggingContext loggingContext = new WorkerLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "RUN1", "INSTANCE1");
    KafkaLogReader logReader = KAFKA_TESTER.getInjector().getInstance(KafkaLogReader.class);
    LoggingTester tester = new LoggingTester();
    tester.testGetNext(logReader, loggingContext);
}
Also used : KafkaLogReader(io.cdap.cdap.logging.read.KafkaLogReader) LoggingTester(io.cdap.cdap.logging.appender.LoggingTester) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) Test(org.junit.Test)

Example 4 with WorkerLoggingContext

use of io.cdap.cdap.logging.context.WorkerLoggingContext in project cdap by caskdata.

the class KafkaLogProcessorPipelineTest method testMetricsAppender.

@Test
public void testMetricsAppender() throws Exception {
    Injector injector = KAFKA_TESTER.getInjector();
    MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
    collectionService.startAndWait();
    LoggerContext loggerContext = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetricsCollectionService.class));
    final File logDir = TEMP_FOLDER.newFolder();
    loggerContext.putProperty("logDirectory", logDir.getAbsolutePath());
    LogPipelineConfigurator configurator = new LogPipelineConfigurator(CConfiguration.create());
    configurator.setContext(loggerContext);
    URL configURL = getClass().getClassLoader().getResource("pipeline-metric-appender.xml");
    Assert.assertNotNull(configURL);
    configurator.doConfigure(configURL);
    String topic = "metricsPipeline";
    TestCheckpointManager checkpointManager = new TestCheckpointManager();
    KafkaPipelineConfig config = new KafkaPipelineConfig(topic, Collections.singleton(0), 1024L, 100L, 1048576, 200L);
    KAFKA_TESTER.createTopic(topic, 1);
    loggerContext.start();
    KafkaLogProcessorPipeline pipeline = new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "testMetricAppender", loggerContext, NO_OP_METRICS_CONTEXT, 0), checkpointManager, KAFKA_TESTER.getBrokerService(), config);
    pipeline.startAndWait();
    // Publish some log messages to Kafka
    long now = System.currentTimeMillis();
    WorkerLoggingContext loggingContext = new WorkerLoggingContext("default", "app1", "worker1", "run1", "instance1");
    publishLog(topic, ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "0", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "2", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "3", now - 500), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "1", now - 900), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.DEBUG, "hidden", now - 600), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "4", now - 100)), loggingContext);
    WorkflowProgramLoggingContext workflowProgramLoggingContext = new WorkflowProgramLoggingContext("default", "app1", "wflow1", "run1", ProgramType.MAPREDUCE, "mr1", "mrun1");
    publishLog(topic, ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.WARN, "0", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.WARN, "2", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.TRACE, "3", now - 500)), workflowProgramLoggingContext);
    ServiceLoggingContext serviceLoggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.TRANSACTION);
    publishLog(topic, ImmutableList.of(LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "0", now - 1000), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "2", now - 700), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.ERROR, "3", now - 500), LogPipelineTestUtil.createLoggingEvent("test.logger", Level.INFO, "1", now - 900)), serviceLoggingContext);
    final MetricStore metricStore = injector.getInstance(MetricStore.class);
    try {
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.info", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(loggingContext), new ArrayList<>()), 5L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.debug", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(loggingContext), new ArrayList<>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.warn", AggregationFunction.SUM, // mapreduce metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.MAPREDUCE, "mr1", Constants.Metrics.Tag.RUN_ID, "mrun1"), new ArrayList<>()), 2L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.trace", AggregationFunction.SUM, // workflow metrics context
        ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.WORKFLOW, "wflow1", Constants.Metrics.Tag.RUN_ID, "run1"), new ArrayList<>()), 1L);
        verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.services.log.error", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(serviceLoggingContext), new ArrayList<>()), 3L);
    } finally {
        pipeline.stopAndWait();
        loggerContext.stop();
        collectionService.stopAndWait();
    }
}
Also used : MetricStore(io.cdap.cdap.api.metrics.MetricStore) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) LocalMetricsCollectionService(io.cdap.cdap.metrics.collect.LocalMetricsCollectionService) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) ArrayList(java.util.ArrayList) LogProcessorPipelineContext(io.cdap.cdap.logging.pipeline.LogProcessorPipelineContext) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) LoggerContext(ch.qos.logback.classic.LoggerContext) URL(java.net.URL) LocationFactory(org.apache.twill.filesystem.LocationFactory) LocalAppenderContext(io.cdap.cdap.logging.framework.LocalAppenderContext) WorkflowProgramLoggingContext(io.cdap.cdap.logging.context.WorkflowProgramLoggingContext) Injector(com.google.inject.Injector) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) MetricDataQuery(io.cdap.cdap.api.metrics.MetricDataQuery) File(java.io.File) LogPipelineConfigurator(io.cdap.cdap.logging.pipeline.LogPipelineConfigurator) Test(org.junit.Test)

Example 5 with WorkerLoggingContext

use of io.cdap.cdap.logging.context.WorkerLoggingContext in project cdap by caskdata.

the class LogBufferRecoveryServiceTest method getLoggingEvents.

private ImmutableList<byte[]> getLoggingEvents() {
    WorkerLoggingContext loggingContext = new WorkerLoggingContext("default", "app1", "worker1", "run1", "instance1");
    long now = System.currentTimeMillis();
    return ImmutableList.of(serializer.toBytes(createLoggingEvent("test.logger", Level.INFO, "0", now - 1000, loggingContext)), serializer.toBytes(createLoggingEvent("test.logger", Level.INFO, "1", now - 900, loggingContext)), serializer.toBytes(createLoggingEvent("test.logger", Level.INFO, "2", now - 700, loggingContext)), serializer.toBytes(createLoggingEvent("test.logger", Level.DEBUG, "3", now - 600, loggingContext)), serializer.toBytes(createLoggingEvent("test.logger", Level.INFO, "4", now - 500, loggingContext)), serializer.toBytes(createLoggingEvent("test.logger", Level.INFO, "5", now - 100, loggingContext)));
}
Also used : WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext)

Aggregations

WorkerLoggingContext (io.cdap.cdap.logging.context.WorkerLoggingContext)18 Test (org.junit.Test)10 LoggingContext (io.cdap.cdap.common.logging.LoggingContext)7 LoggingTester (io.cdap.cdap.logging.appender.LoggingTester)7 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)4 FileLogReader (io.cdap.cdap.logging.read.FileLogReader)4 Logger (org.slf4j.Logger)4 LoggerContext (ch.qos.logback.classic.LoggerContext)3 AbstractModule (com.google.inject.AbstractModule)2 Injector (com.google.inject.Injector)2 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)2 ConfigModule (io.cdap.cdap.common.guice.ConfigModule)2 NonCustomLocationUnitTestModule (io.cdap.cdap.common.guice.NonCustomLocationUnitTestModule)2 DataSetsModules (io.cdap.cdap.data.runtime.DataSetsModules)2 LoggingConfiguration (io.cdap.cdap.logging.LoggingConfiguration)2 LocalLogAppender (io.cdap.cdap.logging.framework.local.LocalLogAppender)2 LocalLogAppenderModule (io.cdap.cdap.logging.guice.LocalLogAppenderModule)2 LogEvent (io.cdap.cdap.logging.read.LogEvent)2 AuthenticationContextModules (io.cdap.cdap.security.auth.context.AuthenticationContextModules)2 AuthorizationEnforcementModule (io.cdap.cdap.security.authorization.AuthorizationEnforcementModule)2