Search in sources :

Example 1 with MockCheckpointManager

use of io.cdap.cdap.logging.logbuffer.MockCheckpointManager in project cdap by caskdata.

the class LogBufferHandlerTest method getLogPipeline.

private LogBufferProcessorPipeline getLogPipeline(LoggerContext loggerContext) {
    MockCheckpointManager checkpointManager = new MockCheckpointManager();
    LogBufferPipelineConfig config = new LogBufferPipelineConfig(1024L, 300L, 500L, 4);
    loggerContext.start();
    return new LogBufferProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0), config, checkpointManager, 0);
}
Also used : MockCheckpointManager(io.cdap.cdap.logging.logbuffer.MockCheckpointManager) LogBufferProcessorPipeline(io.cdap.cdap.logging.pipeline.logbuffer.LogBufferProcessorPipeline) LogBufferPipelineConfig(io.cdap.cdap.logging.pipeline.logbuffer.LogBufferPipelineConfig) LogProcessorPipelineContext(io.cdap.cdap.logging.pipeline.LogProcessorPipelineContext)

Example 2 with MockCheckpointManager

use of io.cdap.cdap.logging.logbuffer.MockCheckpointManager in project cdap by caskdata.

the class LogBufferCleanerTest method testLogBufferCleanerService.

@Test
public void testLogBufferCleanerService() throws Exception {
    String absolutePath = TMP_FOLDER.newFolder().getAbsolutePath();
    MockCheckpointManager checkpointManager = new MockCheckpointManager();
    // update checkpoints
    checkpointManager.saveCheckpoints(ImmutableMap.of(0, new TestCheckpoint(2L, 0L, 1L)));
    // write directly to log buffer, keep file size 10 bytes so that more files are created
    LogBufferWriter writer = new LogBufferWriter(absolutePath, 10, new LogBufferCleaner(ImmutableList.of(checkpointManager), absolutePath, new AtomicBoolean(true)));
    ImmutableList<byte[]> events = getLoggingEvents();
    List<byte[]> subset = new ArrayList<>();
    subset.add(events.get(0));
    subset.add(events.get(1));
    subset.add(events.get(2));
    writer.write(subset.iterator()).iterator();
    // should delete file 0 an1
    File file0 = new File(absolutePath, "0.buff");
    File file1 = new File(absolutePath, "1.buff");
    Tasks.waitFor(true, () -> !file0.exists() && !file1.exists(), 120, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // update checkpoints
    checkpointManager.saveCheckpoints(ImmutableMap.of(0, new TestCheckpoint(5L, 0L, 1L)));
    subset.add(events.get(3));
    subset.add(events.get(4));
    subset.add(events.get(5));
    writer.write(subset.iterator()).iterator();
    writer.close();
    // should delete file 2, 3 and 4
    File file2 = new File(absolutePath, "2.buff");
    File file3 = new File(absolutePath, "3.buff");
    File file4 = new File(absolutePath, "4.buff");
    Tasks.waitFor(true, () -> !file2.exists() && !file3.exists() && !file4.exists(), 120, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MockCheckpointManager(io.cdap.cdap.logging.logbuffer.MockCheckpointManager) ArrayList(java.util.ArrayList) File(java.io.File) LogBufferWriter(io.cdap.cdap.logging.logbuffer.LogBufferWriter) Test(org.junit.Test)

Example 3 with MockCheckpointManager

use of io.cdap.cdap.logging.logbuffer.MockCheckpointManager in project cdap by caskdata.

the class LogBufferRecoveryServiceTest method testLogBufferRecoveryService.

@Test
public void testLogBufferRecoveryService() throws Exception {
    String absolutePath = TMP_FOLDER.newFolder().getAbsolutePath();
    // create and start pipeline
    LoggerContext loggerContext = LogPipelineTestUtil.createLoggerContext("WARN", ImmutableMap.of("test.logger", "INFO"), MockAppender.class.getName());
    final MockAppender appender = LogPipelineTestUtil.getAppender(loggerContext.getLogger(Logger.ROOT_LOGGER_NAME), "Test", MockAppender.class);
    MockCheckpointManager checkpointManager = new MockCheckpointManager();
    LogBufferPipelineConfig config = new LogBufferPipelineConfig(1024L, 300L, 500L, 4);
    loggerContext.start();
    LogBufferProcessorPipeline pipeline = new LogBufferProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0), config, checkpointManager, 0);
    // start the pipeline
    pipeline.startAndWait();
    // write directly to log buffer
    LogBufferWriter writer = new LogBufferWriter(absolutePath, 250, () -> {
    });
    ImmutableList<byte[]> events = getLoggingEvents();
    writer.write(events.iterator()).iterator();
    writer.close();
    // start log buffer reader to read log events from files. keep the batch size as 2 so that there are more than 1
    // iterations
    LogBufferRecoveryService service = new LogBufferRecoveryService(ImmutableList.of(pipeline), ImmutableList.of(checkpointManager), absolutePath, 2, new AtomicBoolean(true));
    service.startAndWait();
    Tasks.waitFor(5, () -> appender.getEvents().size(), 120, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    service.stopAndWait();
    pipeline.stopAndWait();
    loggerContext.stop();
}
Also used : MockAppender(io.cdap.cdap.logging.pipeline.MockAppender) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) MockCheckpointManager(io.cdap.cdap.logging.logbuffer.MockCheckpointManager) LogBufferProcessorPipeline(io.cdap.cdap.logging.pipeline.logbuffer.LogBufferProcessorPipeline) LogBufferPipelineConfig(io.cdap.cdap.logging.pipeline.logbuffer.LogBufferPipelineConfig) LogProcessorPipelineContext(io.cdap.cdap.logging.pipeline.LogProcessorPipelineContext) LoggerContext(ch.qos.logback.classic.LoggerContext) LogBufferWriter(io.cdap.cdap.logging.logbuffer.LogBufferWriter) Test(org.junit.Test)

Aggregations

MockCheckpointManager (io.cdap.cdap.logging.logbuffer.MockCheckpointManager)3 LogBufferWriter (io.cdap.cdap.logging.logbuffer.LogBufferWriter)2 LogProcessorPipelineContext (io.cdap.cdap.logging.pipeline.LogProcessorPipelineContext)2 LogBufferPipelineConfig (io.cdap.cdap.logging.pipeline.logbuffer.LogBufferPipelineConfig)2 LogBufferProcessorPipeline (io.cdap.cdap.logging.pipeline.logbuffer.LogBufferProcessorPipeline)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Test (org.junit.Test)2 LoggerContext (ch.qos.logback.classic.LoggerContext)1 MockAppender (io.cdap.cdap.logging.pipeline.MockAppender)1 File (java.io.File)1 ArrayList (java.util.ArrayList)1