Examples with LogBufferEvent io.cdap.cdap.logging.logbuffer.LogBufferEvent used on opensource projects

Example 1 with LogBufferEvent

use of io.cdap.cdap.logging.logbuffer.LogBufferEvent in project cdap by caskdata.

the class LogBufferProcessorPipelineTest method testSingleAppender.

@Test
public void testSingleAppender() throws Exception {
    LoggerContext loggerContext = LogPipelineTestUtil.createLoggerContext("WARN", ImmutableMap.of("test.logger", "INFO"), MockAppender.class.getName());
    final MockAppender appender = LogPipelineTestUtil.getAppender(loggerContext.getLogger(Logger.ROOT_LOGGER_NAME), "Test", MockAppender.class);
    MockCheckpointManager checkpointManager = new MockCheckpointManager();
    LogBufferPipelineConfig config = new LogBufferPipelineConfig(1024L, 300L, 500L, 4);
    loggerContext.start();
    LogBufferProcessorPipeline pipeline = new LogBufferProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0), config, checkpointManager, 0);
    // start the pipeline
    pipeline.startAndWait();
    // start thread to write to incomingEventQueue
    List<ILoggingEvent> events = getLoggingEvents();
    AtomicInteger i = new AtomicInteger(0);
    List<LogBufferEvent> bufferEvents = events.stream().map(event -> {
        LogBufferEvent lbe = new LogBufferEvent(event, serializer.toBytes(event).length, new LogBufferFileOffset(0, i.get()));
        i.incrementAndGet();
        return lbe;
    }).collect(Collectors.toList());
    // start a thread to send log buffer events to pipeline
    ExecutorService executorService = Executors.newSingleThreadExecutor();
    executorService.execute(() -> {
        for (int count = 0; count < 40; count++) {
            pipeline.processLogEvents(bufferEvents.iterator());
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
            // should not happen
            }
        }
    });
    // wait for pipeline to append all the logs to appender. The DEBUG message should get filtered out.
    Tasks.waitFor(200, () -> appender.getEvents().size(), 60, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    executorService.shutdown();
    pipeline.stopAndWait();
    loggerContext.stop();
}

Example 2 with LogBufferEvent

use of io.cdap.cdap.logging.logbuffer.LogBufferEvent in project cdap by caskdata.

the class LogBufferReaderTest method verifyEvents.

private void verifyEvents(List<LogBufferEvent> logBufferEvents, LogBufferReader reader, Iterator<LogBufferEvent> iterator) throws IOException {
    while (reader.readEvents(logBufferEvents) > 0) {
        for (LogBufferEvent event : logBufferEvents) {
            LogBufferEvent next = iterator.next();
            Assert.assertEquals(next.getOffset(), event.getOffset());
            Assert.assertEquals(next.getEventSize(), event.getEventSize());
            Assert.assertEquals(next.getLogEvent().getMessage(), event.getLogEvent().getMessage());
        }
        logBufferEvents.clear();
    }
    Assert.assertFalse(iterator.hasNext());
}

Example 3 with LogBufferEvent

use of io.cdap.cdap.logging.logbuffer.LogBufferEvent in project cdap by caskdata.

the class LogBufferReaderTest method testLogReader.

@Test
public void testLogReader() throws Exception {
    String absolutePath = TMP_FOLDER.newFolder().getAbsolutePath();
    LogBufferWriter writer = new LogBufferWriter(absolutePath, 250, () -> {
    });
    ImmutableList<byte[]> events = getLoggingEvents();
    Iterable<LogBufferEvent> writtenEvents = writer.write(events.iterator());
    writer.close();
    List<LogBufferEvent> logBufferEvents = new LinkedList<>();
    // read from start positions, tests case where no checkpoints are persisted
    LogBufferReader reader = new LogBufferReader(absolutePath, 2, 3, -1, -1);
    Iterator<LogBufferEvent> iterator = writtenEvents.iterator();
    verifyEvents(logBufferEvents, reader, iterator);
    reader.close();
    // this should skip first and second event, this is because log buffer offsets are offset for event that is
    // already stored. so in this case, skip first event and skip second event as second event is the last stored event
    reader = new LogBufferReader(absolutePath, 2, 3, 0, 145);
    iterator = writtenEvents.iterator();
    iterator.next();
    iterator.next();
    verifyEvents(logBufferEvents, reader, iterator);
    reader.close();
}