Examples with LogOffset io.cdap.cdap.logging.read.LogOffset used on opensource projects

Example 1 with LogOffset

use of io.cdap.cdap.logging.read.LogOffset in project cdap by caskdata.

the class LoggingTester method testGetNext.

public void testGetNext(LogReader logReader, LoggingContext loggingContext) throws Exception {
    LogCallback logCallback1 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> events = logCallback1.getEvents();
    Assert.assertEquals(10, events.size());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(9).getLoggingEvent().getFormattedMessage());
    LogOffset ultimateOffset = events.get(9).getOffset();
    LogOffset penultimateOffset = events.get(8).getOffset();
    LogCallback logCallback2 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback1.getFirstOffset()), 20, Filter.EMPTY_FILTER, logCallback2);
    events = logCallback2.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("Test log message 30 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 49 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback3 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(logCallback2.getLastOffset()), 20, Filter.EMPTY_FILTER, logCallback3);
    events = logCallback3.getEvents();
    Assert.assertEquals(10, events.size());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(9).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback4 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(logCallback2.getFirstOffset()), 20, Filter.EMPTY_FILTER, logCallback4);
    events = logCallback4.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("Test log message 31 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback5 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(ultimateOffset), 20, Filter.EMPTY_FILTER, logCallback5);
    events = logCallback5.getEvents();
    Assert.assertEquals(0, events.size());
    LogCallback logCallback6 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(getNextOffset(ultimateOffset)), 20, Filter.EMPTY_FILTER, logCallback6);
    events = logCallback6.getEvents();
    Assert.assertEquals(0, events.size());
    LogCallback logCallback7 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(penultimateOffset), 20, Filter.EMPTY_FILTER, logCallback7);
    events = logCallback7.getEvents();
    Assert.assertEquals(1, events.size());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    // Try with a different run
    LogCallback logCallback10 = new LogCallback();
    logReader.getLogPrev(replaceTag(loggingContext, new Entry(ApplicationLoggingContext.TAG_RUN_ID, "RUN2")), ReadRange.LATEST, 20, Filter.EMPTY_FILTER, logCallback10);
    events = logCallback10.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("RUN2 Test log message 40 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("RUN2 Test log message 59 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    // Try with a null runid, should return all events with or without runid
    LogCallback logCallback11 = new LogCallback();
    logReader.getLogPrev(replaceTag(loggingContext, new Entry(ApplicationLoggingContext.TAG_RUN_ID, null)), ReadRange.LATEST, 55, Filter.EMPTY_FILTER, logCallback11);
    events = logCallback11.getEvents();
    Assert.assertEquals(55, events.size());
    Assert.assertEquals("RUN2 Test log message 45 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("NULL Test log message 59 arg1 arg2", events.get(34).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("NULL_PARAM Test log message 79 null arg2", events.get(54).getLoggingEvent().getFormattedMessage());
}

Example 2 with LogOffset

use of io.cdap.cdap.logging.read.LogOffset in project cdap by caskdata.

the class AbstractLogHttpHandler method doPrev.

protected void doPrev(LogReader logReader, HttpResponder responder, LoggingContext loggingContext, int maxEvents, String fromOffsetStr, boolean escape, String filterStr, @Nullable RunRecordDetail runRecord, String format, List<String> fieldsToSuppress) {
    try {
        Filter filter = FilterParser.parse(filterStr);
        Callback logCallback = getNextOrPrevLogsCallback(format, responder, fieldsToSuppress, escape);
        LogOffset logOffset = FormattedTextLogEvent.parseLogOffset(fromOffsetStr);
        ReadRange readRange = ReadRange.createToRange(logOffset);
        readRange = adjustReadRange(readRange, runRecord, true);
        try {
            logReader.getLogPrev(loggingContext, readRange, maxEvents, filter, logCallback);
        } catch (Exception ex) {
            LOG.debug("Exception while reading logs for logging context {}", loggingContext, ex);
        } finally {
            logCallback.close();
        }
    } catch (SecurityException e) {
        responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
    } catch (IllegalArgumentException e) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
    }
}

Example 3 with LogOffset

use of io.cdap.cdap.logging.read.LogOffset in project cdap by caskdata.

the class AbstractLogHttpHandler method doNext.

protected void doNext(LogReader logReader, HttpResponder responder, LoggingContext loggingContext, int maxEvents, String fromOffsetStr, boolean escape, String filterStr, @Nullable RunRecordDetail runRecord, String format, List<String> fieldsToSuppress) {
    try {
        Filter filter = FilterParser.parse(filterStr);
        Callback logCallback = getNextOrPrevLogsCallback(format, responder, fieldsToSuppress, escape);
        LogOffset logOffset = FormattedTextLogEvent.parseLogOffset(fromOffsetStr);
        ReadRange readRange = ReadRange.createFromRange(logOffset);
        readRange = adjustReadRange(readRange, runRecord, true);
        try {
            logReader.getLogNext(loggingContext, readRange, maxEvents, filter, logCallback);
        } catch (Exception ex) {
            LOG.debug("Exception while reading logs for logging context {}", loggingContext, ex);
        } finally {
            logCallback.close();
        }
    } catch (SecurityException e) {
        responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
    } catch (IllegalArgumentException e) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
    }
}

Example 4 with LogOffset

use of io.cdap.cdap.logging.read.LogOffset in project cdap by caskdata.

the class LoggingTester method testGetPrev.

public void testGetPrev(LogReader logReader, LoggingContext loggingContext) throws Exception {
    LogCallback logCallback1 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.LATEST, 10, Filter.EMPTY_FILTER, logCallback1);
    List<LogEvent> events = logCallback1.getEvents();
    Assert.assertEquals(10, events.size());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(9).getLoggingEvent().getFormattedMessage());
    LogOffset ultimateOffset = events.get(9).getOffset();
    LogCallback logCallback2 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback1.getFirstOffset()), 20, Filter.EMPTY_FILTER, logCallback2);
    events = logCallback2.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("Test log message 30 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 49 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback3 = new LogCallback();
    logReader.getLogNext(loggingContext, ReadRange.createFromRange(logCallback2.getLastOffset()), 20, Filter.EMPTY_FILTER, logCallback3);
    events = logCallback3.getEvents();
    Assert.assertEquals(10, events.size());
    Assert.assertEquals("Test log message 50 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(9).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback4 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback2.getFirstOffset()), 15, Filter.EMPTY_FILTER, logCallback4);
    events = logCallback4.getEvents();
    // In kafka mode, we'll get only 10 lines, need to run the call again.
    if (events.size() < 15) {
        LogCallback logCallback41 = new LogCallback();
        logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback4.getFirstOffset()), 5, Filter.EMPTY_FILTER, logCallback41);
        events.addAll(0, logCallback41.getEvents());
        logCallback4 = logCallback41;
    }
    Assert.assertEquals(15, events.size());
    Assert.assertEquals("Test log message 15 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 29 arg1 arg2", events.get(14).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback6 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback4.getFirstOffset()), 25, Filter.EMPTY_FILTER, logCallback6);
    events = logCallback6.getEvents();
    Assert.assertEquals(15, events.size());
    Assert.assertEquals("Test log message 0 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 14 arg1 arg2", events.get(14).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback5 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback6.getFirstOffset()), 15, Filter.EMPTY_FILTER, logCallback5);
    events = logCallback5.getEvents();
    Assert.assertEquals(0, events.size());
    LogCallback logCallback7 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(logCallback4.getFirstOffset()), 15, Filter.EMPTY_FILTER, logCallback7);
    events = logCallback7.getEvents();
    Assert.assertEquals(15, events.size());
    Assert.assertEquals("Test log message 0 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 14 arg1 arg2", events.get(14).getLoggingEvent().getFormattedMessage());
    LogCallback logCallback9 = new LogCallback();
    logReader.getLogPrev(loggingContext, ReadRange.createToRange(getNextOffset(ultimateOffset)), 15, Filter.EMPTY_FILTER, logCallback9);
    events = logCallback9.getEvents();
    Assert.assertEquals(15, events.size());
    Assert.assertEquals("Test log message 45 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("Test log message 59 arg1 arg2", events.get(14).getLoggingEvent().getFormattedMessage());
    // Try with a different run
    LogCallback logCallback10 = new LogCallback();
    logReader.getLogPrev(replaceTag(loggingContext, new Entry(ApplicationLoggingContext.TAG_RUN_ID, "RUN2")), ReadRange.LATEST, 20, Filter.EMPTY_FILTER, logCallback10);
    events = logCallback10.getEvents();
    Assert.assertEquals(20, events.size());
    Assert.assertEquals("RUN2 Test log message 40 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("RUN2 Test log message 59 arg1 arg2", events.get(19).getLoggingEvent().getFormattedMessage());
    // Try with a null runid, should return all events with or without runid
    LogCallback logCallback11 = new LogCallback();
    logReader.getLogPrev(replaceTag(loggingContext, new Entry(ApplicationLoggingContext.TAG_RUN_ID, null)), ReadRange.LATEST, 60, Filter.EMPTY_FILTER, logCallback11);
    events = logCallback11.getEvents();
    Assert.assertEquals(60, events.size());
    Assert.assertEquals("RUN2 Test log message 40 arg1 arg2", events.get(0).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("NULL Test log message 59 arg1 arg2", events.get(39).getLoggingEvent().getFormattedMessage());
    Assert.assertEquals("NULL_PARAM Test log message 79 null arg2", events.get(59).getLoggingEvent().getFormattedMessage());
}

Example 5 with LogOffset

use of io.cdap.cdap.logging.read.LogOffset in project cdap by caskdata.

the class MockLogReader method generateLogs.

/**
 * This method is used to generate the logs for program which are used for testing.
 * Single call to this method would add {@link #MAX} number of events.
 * First 20 events are generated without {@link ApplicationLoggingContext#TAG_RUN_ID} tag.
 * For next 40 events, alternate event is tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
 * Last 20 events are not tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
 * All events are alternately marked as {@link Level#ERROR} and {@link Level#WARN}.
 * All events are alternately tagged with "plugin", "program" and "system" as value of MDC property ".origin"
 * All events are alternately tagged with "lifecycle" as value of MDC property "MDC:eventType
 */
private void generateLogs(LoggingContext loggingContext, ProgramId programId, ProgramRunStatus runStatus) throws InterruptedException {
    // All possible values of " MDC property ".origin
    String[] origins = { "plugin", "program", "system" };
    String entityId = LoggingContextHelper.getEntityId(loggingContext).getValue();
    StackTraceElement stackTraceElementNative = new StackTraceElement("io.cdap.Test", "testMethod", null, -2);
    RunId runId = null;
    Long stopTs = null;
    for (int i = 0; i < MAX; ++i) {
        // Setup run id for event with ids >= 20
        if (i == 20) {
            runId = RunIds.generate(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        } else if (i == 60 && runStatus != ProgramRunStatus.RUNNING && runStatus != ProgramRunStatus.SUSPENDED) {
            // Record stop time for run for 60th event, but still continue to record run in the other logging events.
            stopTs = getMockTimeSecs(i);
        }
        LoggingEvent event = new LoggingEvent("io.cdap.Test", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), i % 2 == 0 ? Level.ERROR : Level.WARN, entityId + "<img>-" + i, null, null);
        event.setTimeStamp(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        // Add runid to logging context
        Map<String, String> tagMap = Maps.newHashMap(Maps.transformValues(loggingContext.getSystemTagsMap(), TAG_TO_STRING_FUNCTION));
        if (runId != null && stopTs == null && i % 2 == 0) {
            tagMap.put(ApplicationLoggingContext.TAG_RUN_ID, runId.getId());
        }
        // Determine the value of ".origin" property by (i % 3)
        tagMap.put(".origin", origins[i % 3]);
        if (i % 2 == 0) {
            tagMap.put("MDC:eventType", "lifecycle");
        }
        if (i == 30) {
            event.setCallerData(new StackTraceElement[] { stackTraceElementNative });
        }
        event.setMDCPropertyMap(tagMap);
        logEvents.add(new LogEvent(event, new LogOffset(i, i)));
    }
    long startTs = RunIds.getTime(runId, TimeUnit.SECONDS);
    if (programId != null) {
        // noinspection ConstantConditions
        runRecordMap.put(programId, RunRecord.builder().setRunId(runId.getId()).setStartTime(startTs).setRunTime(startTs + 1).setStopTime(stopTs).setStatus(runStatus).setCluster(new ProgramRunCluster(ProgramRunClusterStatus.PROVISIONED, null, null)).build());
        setStartAndRunning(programId.run(runId.getId()));
        if (stopTs != null) {
            store.setStop(programId.run(runId.getId()), stopTs, runStatus, AppFabricTestHelper.createSourceId(++sourceId));
        }
    }
}