Examples with ReadRange io.cdap.cdap.logging.read.ReadRange used on opensource projects

Example 11 with ReadRange

use of io.cdap.cdap.logging.read.ReadRange in project cdap by cdapio.

the class TestDistributedLogReader method testDistributedLogNextFile.

@Test
public void testDistributedLogNextFile() throws Exception {
    ReadRange readRange = new ReadRange(0, Long.MAX_VALUE, LogOffset.INVALID_KAFKA_OFFSET);
    testDistributedLogNext(readRange, LOGGING_CONTEXT_FILE, 14, 3, "TestDistributedLogReader Log message2 ", 40, 0);
    readRange = new ReadRange(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1), System.currentTimeMillis(), LogOffset.INVALID_KAFKA_OFFSET);
    testDistributedLogNext(readRange, LOGGING_CONTEXT_FILE, 14, 3, "TestDistributedLogReader Log message2 ", 40, 0);
    testDistributedLogNext(ReadRange.LATEST, LOGGING_CONTEXT_FILE, 1, 5, "TestDistributedLogReader Log message2 ", 5, 35);
}

Example 12 with ReadRange

use of io.cdap.cdap.logging.read.ReadRange in project cdap by cdapio.

the class TestDistributedLogReader method testDistributedLogPrevBoth.

@Test
public void testDistributedLogPrevBoth() throws Exception {
    ReadRange readRange = new ReadRange(0, Long.MAX_VALUE, LogOffset.INVALID_KAFKA_OFFSET);
    testDistributedLogPrev(readRange, LOGGING_CONTEXT_BOTH, 16, 4, "TestDistributedLogReader Log message1 ", 60);
    readRange = new ReadRange(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(1), System.currentTimeMillis(), LogOffset.INVALID_KAFKA_OFFSET);
    testDistributedLogPrev(readRange, LOGGING_CONTEXT_BOTH, 16, 4, "TestDistributedLogReader Log message1 ", 60);
    testDistributedLogPrev(ReadRange.LATEST, LOGGING_CONTEXT_BOTH, 9, 8, "TestDistributedLogReader Log message1 ", 60);
}

Example 13 with ReadRange

use of io.cdap.cdap.logging.read.ReadRange in project cdap by cdapio.

the class AbstractLogHttpHandler method doGetLogs.

protected void doGetLogs(LogReader logReader, HttpResponder responder, LoggingContext loggingContext, long fromTimeSecsParam, long toTimeSecsParam, boolean escape, String filterStr, @Nullable RunRecordDetail runRecord, String format, List<String> fieldsToSuppress) {
    try {
        TimeRange timeRange = parseTime(fromTimeSecsParam, toTimeSecsParam, responder);
        if (timeRange == null) {
            return;
        }
        Filter filter = FilterParser.parse(filterStr);
        ReadRange readRange = new ReadRange(timeRange.getFromMillis(), timeRange.getToMillis(), LogOffset.INVALID_KAFKA_OFFSET);
        readRange = adjustReadRange(readRange, runRecord, fromTimeSecsParam != -1);
        try {
            // the iterator is closed by the BodyProducer passed to the HttpResponder
            CloseableIterator<LogEvent> logIter = logReader.getLog(loggingContext, readRange.getFromMillis(), readRange.getToMillis(), filter);
            AbstractChunkedLogProducer logsProducer = getFullLogsProducer(format, logIter, fieldsToSuppress, escape);
            responder.sendContent(HttpResponseStatus.OK, logsProducer, logsProducer.getResponseHeaders());
        } catch (Exception ex) {
            LOG.debug("Exception while reading logs for logging context {}", loggingContext, ex);
            responder.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
        }
    } catch (SecurityException e) {
        responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
    } catch (IllegalArgumentException e) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
    }
}

Example 14 with ReadRange

use of io.cdap.cdap.logging.read.ReadRange in project cdap by cdapio.

the class AbstractLogHttpHandler method doPrev.

protected void doPrev(LogReader logReader, HttpResponder responder, LoggingContext loggingContext, int maxEvents, String fromOffsetStr, boolean escape, String filterStr, @Nullable RunRecordDetail runRecord, String format, List<String> fieldsToSuppress) {
    try {
        Filter filter = FilterParser.parse(filterStr);
        Callback logCallback = getNextOrPrevLogsCallback(format, responder, fieldsToSuppress, escape);
        LogOffset logOffset = FormattedTextLogEvent.parseLogOffset(fromOffsetStr);
        ReadRange readRange = ReadRange.createToRange(logOffset);
        readRange = adjustReadRange(readRange, runRecord, true);
        try {
            logReader.getLogPrev(loggingContext, readRange, maxEvents, filter, logCallback);
        } catch (Exception ex) {
            LOG.debug("Exception while reading logs for logging context {}", loggingContext, ex);
        } finally {
            logCallback.close();
        }
    } catch (SecurityException e) {
        responder.sendStatus(HttpResponseStatus.UNAUTHORIZED);
    } catch (IllegalArgumentException e) {
        responder.sendString(HttpResponseStatus.BAD_REQUEST, e.getMessage());
    }
}

Example 15 with ReadRange

use of io.cdap.cdap.logging.read.ReadRange in project cdap by cdapio.

the class MockLogReader method getLogPrev.

@Override
public void getLogPrev(LoggingContext loggingContext, ReadRange readRange, int maxEvents, Filter filter, Callback callback) {
    if (readRange.getKafkaOffset() < 0) {
        readRange = new ReadRange(readRange.getFromMillis(), readRange.getToMillis(), MAX);
    }
    Filter contextFilter = LoggingContextHelper.createFilter(loggingContext);
    callback.init();
    try {
        int count = 0;
        long startOffset = readRange.getKafkaOffset() - maxEvents;
        for (LogEvent logLine : logEvents) {
            long logTime = logLine.getLoggingEvent().getTimeStamp();
            if (!contextFilter.match(logLine.getLoggingEvent()) || logTime < readRange.getFromMillis() || logTime >= readRange.getToMillis()) {
                continue;
            }
            if (logLine.getOffset().getKafkaOffset() >= startOffset && logLine.getOffset().getKafkaOffset() < readRange.getKafkaOffset()) {
                if (++count > maxEvents) {
                    break;
                }
                if (filter != Filter.EMPTY_FILTER && logLine.getOffset().getKafkaOffset() % 2 != 0) {
                    continue;
                }
                callback.handle(logLine);
            }
        }
    } catch (Throwable e) {
        LOG.error("Got exception", e);
    } finally {
        callback.close();
    }
}