Examples with LogBufferProcessorPipeline io.cdap.cdap.logging.pipeline.logbuffer.LogBufferProcessorPipeline used on opensource projects

Example 6 with LogBufferProcessorPipeline

use of io.cdap.cdap.logging.pipeline.logbuffer.LogBufferProcessorPipeline in project cdap by caskdata.

the class LogBufferService method loadLogPipelines.

/**
 * Load log buffer pipelines.
 */
@SuppressWarnings("unchecked")
private List<LogBufferProcessorPipeline> loadLogPipelines() {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    List<LogBufferProcessorPipeline> bufferPipelines = new ArrayList<>();
    // Create one LogBufferProcessorPipeline per spec
    for (LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        CConfiguration cConf = pipelineSpec.getConf();
        AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf);
        LogBufferPipelineConfig config = new LogBufferPipelineConfig(bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS), cConf.getLong(Constants.LogBuffer.LOG_BUFFER_PIPELINE_BATCH_SIZE, 1000));
        CheckpointManager checkpointManager = checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix(), CheckpointManagerFactory.Type.LOG_BUFFER);
        LogBufferProcessorPipeline pipeline = new LogBufferProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), config, checkpointManager, 0);
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> pipeline, retryStrategy));
        bufferPipelines.add(pipeline);
        checkpointManagers.add(checkpointManager);
    }
    return bufferPipelines;
}

Example 7 with LogBufferProcessorPipeline

use of io.cdap.cdap.logging.pipeline.logbuffer.LogBufferProcessorPipeline in project cdap by caskdata.

the class LogBufferRecoveryServiceTest method testLogBufferRecoveryService.

@Test
public void testLogBufferRecoveryService() throws Exception {
    String absolutePath = TMP_FOLDER.newFolder().getAbsolutePath();
    // create and start pipeline
    LoggerContext loggerContext = LogPipelineTestUtil.createLoggerContext("WARN", ImmutableMap.of("test.logger", "INFO"), MockAppender.class.getName());
    final MockAppender appender = LogPipelineTestUtil.getAppender(loggerContext.getLogger(Logger.ROOT_LOGGER_NAME), "Test", MockAppender.class);
    MockCheckpointManager checkpointManager = new MockCheckpointManager();
    LogBufferPipelineConfig config = new LogBufferPipelineConfig(1024L, 300L, 500L, 4);
    loggerContext.start();
    LogBufferProcessorPipeline pipeline = new LogBufferProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "test", loggerContext, NO_OP_METRICS_CONTEXT, 0), config, checkpointManager, 0);
    // start the pipeline
    pipeline.startAndWait();
    // write directly to log buffer
    LogBufferWriter writer = new LogBufferWriter(absolutePath, 250, () -> {
    });
    ImmutableList<byte[]> events = getLoggingEvents();
    writer.write(events.iterator()).iterator();
    writer.close();
    // start log buffer reader to read log events from files. keep the batch size as 2 so that there are more than 1
    // iterations
    LogBufferRecoveryService service = new LogBufferRecoveryService(ImmutableList.of(pipeline), ImmutableList.of(checkpointManager), absolutePath, 2, new AtomicBoolean(true));
    service.startAndWait();
    Tasks.waitFor(5, () -> appender.getEvents().size(), 120, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    service.stopAndWait();
    pipeline.stopAndWait();
    loggerContext.stop();
}