Examples with FlowletLoggingContext co.cask.cdap.logging.context.FlowletLoggingContext used on opensource projects

Example 6 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class TestFileLogging method setUpContext.

@BeforeClass
public static void setUpContext() throws Exception {
    Configuration hConf = HBaseConfiguration.create();
    final CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setInt(LoggingConfiguration.LOG_MAX_FILE_SIZE_BYTES, 20 * 1024);
    String logBaseDir = cConf.get(LoggingConfiguration.LOG_BASE_DIR) + "/" + TestFileLogging.class.getSimpleName();
    cConf.set(LoggingConfiguration.LOG_BASE_DIR, logBaseDir);
    injector = Guice.createInjector(new ConfigModule(cConf, hConf), new NonCustomLocationUnitTestModule().getModule(), new TransactionModules().getInMemoryModules(), new LoggingModules().getInMemoryModules(), new DataSetsModules().getInMemoryModules(), new SystemDatasetRuntimeModule().getInMemoryModules(), new AuthorizationTestModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new AuthenticationContextModules().getNoOpModule(), new AbstractModule() {

        @Override
        protected void configure() {
            bind(MetricsCollectionService.class).to(NoOpMetricsCollectionService.class);
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
            bind(UGIProvider.class).to(UnsupportedUGIProvider.class);
            bind(NamespaceQueryAdmin.class).to(SimpleNamespaceQueryAdmin.class);
        }
    });
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    LogAppender appender = injector.getInstance(LocalLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestFileLogging");
    Logger logger = LoggerFactory.getLogger("TestFileLogging");
    LoggingTester loggingTester = new LoggingTester();
    loggingTester.generateLogs(logger, new FlowletLoggingContext("TFL_NS_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN1", "INSTANCE1"));
    appender.stop();
}

Example 7 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class TestFileLogging method testGetLog.

@Test
public void testGetLog() throws Exception {
    // LogReader.getLog is tested in LogSaverTest for distributed mode
    LoggingContext loggingContext = new FlowletLoggingContext("TFL_NS_1", "APP_1", "FLOW_1", "", "RUN1", "INSTANCE1");
    FileLogReader logTail = injector.getInstance(FileLogReader.class);
    LoggingTester.LogCallback logCallback1 = new LoggingTester.LogCallback();
    logTail.getLogPrev(loggingContext, ReadRange.LATEST, 60, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> allEvents = logCallback1.getEvents();
    Assert.assertEquals(60, allEvents.size());
    List<LogEvent> events = Lists.newArrayList(logTail.getLog(loggingContext, allEvents.get(10).getLoggingEvent().getTimeStamp(), allEvents.get(15).getLoggingEvent().getTimeStamp(), Filter.EMPTY_FILTER));
    Assert.assertEquals(5, events.size());
    Assert.assertEquals(allEvents.get(10).getLoggingEvent().getFormattedMessage(), events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals(allEvents.get(14).getLoggingEvent().getFormattedMessage(), events.get(4).getLoggingEvent().getFormattedMessage());
    events = Lists.newArrayList(logTail.getLog(loggingContext, allEvents.get(0).getLoggingEvent().getTimeStamp(), allEvents.get(59).getLoggingEvent().getTimeStamp(), Filter.EMPTY_FILTER));
    Assert.assertEquals(59, events.size());
    Assert.assertEquals(allEvents.get(0).getLoggingEvent().getFormattedMessage(), events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals(allEvents.get(58).getLoggingEvent().getFormattedMessage(), events.get(58).getLoggingEvent().getFormattedMessage());
    events = Lists.newArrayList(logTail.getLog(loggingContext, allEvents.get(12).getLoggingEvent().getTimeStamp(), allEvents.get(41).getLoggingEvent().getTimeStamp(), Filter.EMPTY_FILTER));
    Assert.assertEquals(29, events.size());
    Assert.assertEquals(allEvents.get(12).getLoggingEvent().getFormattedMessage(), events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals(allEvents.get(40).getLoggingEvent().getFormattedMessage(), events.get(28).getLoggingEvent().getFormattedMessage());
    events = Lists.newArrayList(logTail.getLog(loggingContext, allEvents.get(22).getLoggingEvent().getTimeStamp(), allEvents.get(38).getLoggingEvent().getTimeStamp(), Filter.EMPTY_FILTER));
    Assert.assertEquals(16, events.size());
    Assert.assertEquals(allEvents.get(22).getLoggingEvent().getFormattedMessage(), events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals(allEvents.get(37).getLoggingEvent().getFormattedMessage(), events.get(15).getLoggingEvent().getFormattedMessage());
    events = Lists.newArrayList(logTail.getLog(loggingContext, allEvents.get(41).getLoggingEvent().getTimeStamp(), allEvents.get(59).getLoggingEvent().getTimeStamp(), Filter.EMPTY_FILTER));
    Assert.assertEquals(18, events.size());
    Assert.assertEquals(allEvents.get(41).getLoggingEvent().getFormattedMessage(), events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals(allEvents.get(58).getLoggingEvent().getFormattedMessage(), events.get(17).getLoggingEvent().getFormattedMessage());
    // Try with null run id, should get all logs for FLOW_1
    LoggingContext loggingContext1 = new FlowletLoggingContext("TFL_NS_1", "APP_1", "FLOW_1", "", null, "INSTANCE1");
    events = Lists.newArrayList(logTail.getLog(loggingContext1, 0, Long.MAX_VALUE, Filter.EMPTY_FILTER));
    Assert.assertEquals(100, events.size());
}

Example 8 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class TestKafkaLogging method testGetPrev.

@Test
public void testGetPrev() throws Exception {
    LoggingContext loggingContext = new FlowletLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "", "RUN1", "INSTANCE1");
    KafkaLogReader logReader = KAFKA_TESTER.getInjector().getInstance(KafkaLogReader.class);
    LoggingTester tester = new LoggingTester();
    tester.testGetPrev(logReader, loggingContext);
}

Example 9 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class TestKafkaLogging method init.

@BeforeClass
public static void init() throws Exception {
    txManager = KAFKA_TESTER.getInjector().getInstance(TransactionManager.class);
    txManager.startAndWait();
    KafkaLogAppender appender = KAFKA_TESTER.getInjector().getInstance(KafkaLogAppender.class);
    new LogAppenderInitializer(appender).initialize("TestKafkaLogging");
    Logger logger = LoggerFactory.getLogger("TestKafkaLogging");
    LoggingTester loggingTester = new LoggingTester();
    loggingTester.generateLogs(logger, new FlowletLoggingContext("TKL_NS_1", "APP_1", "FLOW_1", "FLOWLET_1", "RUN1", "INSTANCE1"));
    appender.stop();
}

Example 10 with FlowletLoggingContext

use of co.cask.cdap.logging.context.FlowletLoggingContext in project cdap by caskdata.

the class TestKafkaLogging method testPartitionKey.

// Note: LogReader.getLog is tested in LogSaverTest for distributed mode
@Test
public void testPartitionKey() throws Exception {
    CConfiguration cConf = KAFKA_TESTER.getCConf();
    // set kafka partition key to application
    cConf.set(Constants.Logging.LOG_PUBLISH_PARTITION_KEY, "application");
    Logger logger = LoggerFactory.getLogger("TestKafkaLogging");
    LoggingContext loggingContext = new FlowletLoggingContext("TKL_NS_2", "APP_2", "FLOW_2", "FLOWLET_2", "RUN2", "INSTANCE2");
    LoggingContextAccessor.setLoggingContext(loggingContext);
    for (int i = 0; i < 40; ++i) {
        logger.warn("TKL_NS_2 Test log message {} {} {}", i, "arg1", "arg2", new Exception("test exception"));
    }
    loggingContext = new FlowletLoggingContext("TKL_NS_2", "APP_2", "FLOW_3", "FLOWLET_3", "RUN3", "INSTANCE3");
    LoggingContextAccessor.setLoggingContext(loggingContext);
    for (int i = 0; i < 40; ++i) {
        logger.warn("TKL_NS_2 Test log message {} {} {}", i, "arg1", "arg2", new Exception("test exception"));
    }
    final Multimap<Integer, String> actual = ArrayListMultimap.create();
    KAFKA_TESTER.getPublishedMessages(KAFKA_TESTER.getCConf().get(Constants.Logging.KAFKA_TOPIC), ImmutableSet.of(0, 1), 40, new Function<FetchedMessage, String>() {

        @Override
        public String apply(final FetchedMessage input) {
            try {
                Map.Entry<Integer, String> entry = convertFetchedMessage(input);
                actual.put(entry.getKey(), entry.getValue());
            } catch (IOException e) {
            // should never happen
            }
            return "";
        }
    });
    boolean isPresent = false;
    // check if all the logs from same app went to same partition
    for (Map.Entry<Integer, Collection<String>> entry : actual.asMap().entrySet()) {
        if (entry.getValue().contains("TKL_NS_2:APP_2")) {
            if (isPresent) {
                // if we have already found another partition with application context, assert false
                Assert.assertFalse("Only one partition should have application logging context", isPresent);
            }
            isPresent = true;
        }
    }
    // reset kafka partition key
    cConf.set(Constants.Logging.LOG_PUBLISH_PARTITION_KEY, "program");
}