Examples with LogEvent io.cdap.cdap.logging.read.LogEvent used on opensource projects

Example 1 with LogEvent

use of io.cdap.cdap.logging.read.LogEvent in project cdap by caskdata.

the class MockLogReader method getLog.

@Override
public CloseableIterator<LogEvent> getLog(LoggingContext loggingContext, long fromTimeMs, long toTimeMs, Filter filter) {
    CollectingCallback collectingCallback = new CollectingCallback();
    // since its just for test case, we don't need to lazily read logs (which is the purpose of returning an Iterator)
    long fromOffset = getOffset(fromTimeMs / 1000);
    long toOffset = getOffset(toTimeMs / 1000);
    getLogNext(loggingContext, new ReadRange(fromTimeMs, toTimeMs, fromOffset), (int) (toOffset - fromOffset), filter, collectingCallback);
    final Iterator<LogEvent> iterator = collectingCallback.getLogEvents().iterator();
    return new CloseableIterator<LogEvent>() {

        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public LogEvent next() {
            return iterator.next();
        }

        @Override
        public void remove() {
            iterator.remove();
        }

        @Override
        public void close() {
        // no-op
        }
    };
}

Example 2 with LogEvent

use of io.cdap.cdap.logging.read.LogEvent in project cdap by caskdata.

the class MockLogReader method getLogPrev.

@Override
public void getLogPrev(LoggingContext loggingContext, ReadRange readRange, int maxEvents, Filter filter, Callback callback) {
    if (readRange.getKafkaOffset() < 0) {
        readRange = new ReadRange(readRange.getFromMillis(), readRange.getToMillis(), MAX);
    }
    Filter contextFilter = LoggingContextHelper.createFilter(loggingContext);
    callback.init();
    try {
        int count = 0;
        long startOffset = readRange.getKafkaOffset() - maxEvents;
        for (LogEvent logLine : logEvents) {
            long logTime = logLine.getLoggingEvent().getTimeStamp();
            if (!contextFilter.match(logLine.getLoggingEvent()) || logTime < readRange.getFromMillis() || logTime >= readRange.getToMillis()) {
                continue;
            }
            if (logLine.getOffset().getKafkaOffset() >= startOffset && logLine.getOffset().getKafkaOffset() < readRange.getKafkaOffset()) {
                if (++count > maxEvents) {
                    break;
                }
                if (filter != Filter.EMPTY_FILTER && logLine.getOffset().getKafkaOffset() % 2 != 0) {
                    continue;
                }
                callback.handle(logLine);
            }
        }
    } catch (Throwable e) {
        LOG.error("Got exception", e);
    } finally {
        callback.close();
    }
}

Example 3 with LogEvent

use of io.cdap.cdap.logging.read.LogEvent in project cdap by caskdata.

the class LoggingTester method testGetNext.

public void testGetNext(LogReader logReader, LoggingContext loggingContext) throws Exception {
    LogCallback logCallback1 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> events = logCallback1.getEvents();
    Assert.assertEquals(10, events.size());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(9).getLoggingEvent().getFormattedMessage());
    LogOffset ultimateOffset = events.get(9).getOffset();
    LogOffset penultimateOffset = events.get(8).getOffset();
    LogCallback logCallback2 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback1.getFirstOffset()), 20, Filter.EMPTY_FILTER, logCallback2);
    events = logCallback2.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("Test log message 30 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 49 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback3 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(logCallback2.getLastOffset()), 20, Filter.EMPTY_FILTER, logCallback3);
    events = logCallback3.getEvents();
    Assert.assertEquals(10, events.size());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(9).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback4 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(logCallback2.getFirstOffset()), 20, Filter.EMPTY_FILTER, logCallback4);
    events = logCallback4.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("Test log message 31 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback5 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(ultimateOffset), 20, Filter.EMPTY_FILTER, logCallback5);
    events = logCallback5.getEvents();
    Assert.assertEquals(0, events.size());
    LogCallback logCallback6 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(getNextOffset(ultimateOffset)), 20, Filter.EMPTY_FILTER, logCallback6);
    events = logCallback6.getEvents();
    Assert.assertEquals(0, events.size());
    LogCallback logCallback7 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(penultimateOffset), 20, Filter.EMPTY_FILTER, logCallback7);
    events = logCallback7.getEvents();
    Assert.assertEquals(1, events.size());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    // Try with a different run
    LogCallback logCallback10 = new LogCallback();
    logReader.getLogPrev(replaceTag(loggingContext, new Entry(ApplicationLoggingContext.TAG_RUN_ID, "RUN2")), ReadRange.LATEST, 20, Filter.EMPTY_FILTER, logCallback10);
    events = logCallback10.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("RUN2 Test log message 40 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("RUN2 Test log message 59 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    // Try with a null runid, should return all events with or without runid
    LogCallback logCallback11 = new LogCallback();
    logReader.getLogPrev(replaceTag(loggingContext, new Entry(ApplicationLoggingContext.TAG_RUN_ID, null)), ReadRange.LATEST, 55, Filter.EMPTY_FILTER, logCallback11);
    events = logCallback11.getEvents();
    Assert.assertEquals(55, events.size());
    Assert.assertEquals("RUN2 Test log message 45 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("NULL Test log message 59 arg1 arg2", events.get(34).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("NULL_PARAM Test log message 79 null arg2", events.get(54).getLoggingEvent().getFormattedMessage());
}

Example 4 with LogEvent

use of io.cdap.cdap.logging.read.LogEvent in project cdap by caskdata.

the class CDAPLogAppenderTest method testCDAPLogAppender.

@Test
public void testCDAPLogAppender() {
    int syncInterval = 1024 * 1024;
    CDAPLogAppender cdapLogAppender = new CDAPLogAppender();
    cdapLogAppender.setSyncIntervalBytes(syncInterval);
    cdapLogAppender.setMaxFileLifetimeMs(TimeUnit.DAYS.toMillis(1));
    cdapLogAppender.setMaxFileSizeInBytes(104857600);
    cdapLogAppender.setDirPermissions("700");
    cdapLogAppender.setFilePermissions("600");
    cdapLogAppender.setFileRetentionDurationDays(1);
    cdapLogAppender.setLogCleanupIntervalMins(10);
    cdapLogAppender.setFileCleanupBatchSize(100);
    AppenderContext context = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), new NoOpMetricsCollectionService());
    context.start();
    cdapLogAppender.setContext(context);
    cdapLogAppender.start();
    FileMetaDataReader fileMetaDataReader = injector.getInstance(FileMetaDataReader.class);
    LoggingEvent event = new LoggingEvent("io.cdap.Test", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message", null, null);
    Map<String, String> properties = new HashMap<>();
    properties.put(NamespaceLoggingContext.TAG_NAMESPACE_ID, "default");
    properties.put(ApplicationLoggingContext.TAG_APPLICATION_ID, "testApp");
    properties.put(UserServiceLoggingContext.TAG_USER_SERVICE_ID, "testService");
    event.setMDCPropertyMap(properties);
    cdapLogAppender.doAppend(event);
    cdapLogAppender.stop();
    context.stop();
    try {
        List<LogLocation> files = fileMetaDataReader.listFiles(cdapLogAppender.getLoggingPath(properties), 0, Long.MAX_VALUE);
        Assert.assertEquals(1, files.size());
        LogLocation logLocation = files.get(0);
        Assert.assertEquals(LogLocation.VERSION_1, logLocation.getFrameworkVersion());
        Assert.assertTrue(logLocation.getLocation().exists());
        CloseableIterator<LogEvent> logEventCloseableIterator = logLocation.readLog(Filter.EMPTY_FILTER, 0, Long.MAX_VALUE, Integer.MAX_VALUE);
        int logCount = 0;
        while (logEventCloseableIterator.hasNext()) {
            logCount++;
            LogEvent logEvent = logEventCloseableIterator.next();
            Assert.assertEquals(event.getMessage(), logEvent.getLoggingEvent().getMessage());
        }
        logEventCloseableIterator.close();
        Assert.assertEquals(1, logCount);
        // checking permission
        String expectedPermissions = "rw-------";
        for (LogLocation file : files) {
            Location location = file.getLocation();
            Assert.assertEquals(expectedPermissions, location.getPermissions());
        }
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 5 with LogEvent

use of io.cdap.cdap.logging.read.LogEvent in project cdap by caskdata.

the class TestDistributedLogReader method testDistributedLogNext.

public void testDistributedLogNext(ReadRange readRange, LoggingContext loggingContext, int numCalls, int step, String assertMessage, int assertCount, int startIndex) throws Exception {
    DistributedLogReader distributedLogReader = injector.getInstance(DistributedLogReader.class);
    int count = 0;
    for (int i = 0; i < numCalls; ++i) {
        LoggingTester.LogCallback callback = new LoggingTester.LogCallback();
        distributedLogReader.getLogNext(loggingContext, readRange, step, Filter.EMPTY_FILTER, callback);
        List<LogEvent> events = callback.getEvents();
        Assert.assertFalse(events.isEmpty());
        readRange = ReadRange.createFromRange(events.get(events.size() - 1).getOffset());
        for (LogEvent event : events) {
            Assert.assertEquals(assertMessage + (startIndex + count++), event.getLoggingEvent().getFormattedMessage());
            // All logs generated by this test have value "system" in MDC property ".origin"
            Assert.assertEquals("system", event.getLoggingEvent().getMDCPropertyMap().get(".origin"));
        }
    }
    Assert.assertEquals(assertCount, count);
}