Examples with AppenderContext io.cdap.cdap.api.logging.AppenderContext used on opensource projects

Example 11 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by cdapio.

the class CDAPLogAppender method start.

@Override
public void start() {
    // These should all passed. The settings are from the cdap-log-pipeline.xml and the context must be AppenderContext
    Preconditions.checkState(dirPermissions != null, "Property dirPermissions cannot be null");
    Preconditions.checkState(filePermissions != null, "Property filePermissions cannot be null");
    Preconditions.checkState(syncIntervalBytes > 0, "Property syncIntervalBytes must be > 0.");
    Preconditions.checkState(maxFileLifetimeMs > 0, "Property maxFileLifetimeMs must be > 0");
    Preconditions.checkState(maxFileSizeInBytes > 0, "Property maxFileSizeInBytes must be > 0");
    Preconditions.checkState(fileRetentionDurationDays > 0, "Property fileRetentionDurationDays must be > 0");
    Preconditions.checkState(logCleanupIntervalMins > 0, "Property logCleanupIntervalMins must be > 0");
    Preconditions.checkState(fileCleanupBatchSize > 0, "Property fileCleanupBatchSize must be > 0");
    if (context instanceof AppenderContext) {
        AppenderContext context = (AppenderContext) this.context;
        logFileManager = new LogFileManager(dirPermissions, filePermissions, maxFileLifetimeMs, maxFileSizeInBytes, syncIntervalBytes, new FileMetaDataWriter(context.getTransactionRunner()), context.getLocationFactory());
        if (context.getInstanceId() == 0) {
            scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(Threads.createDaemonThreadFactory("log-clean-up"));
            FileMetadataCleaner fileMetadataCleaner = new FileMetadataCleaner(context.getTransactionRunner());
            LogCleaner logCleaner = new LogCleaner(fileMetadataCleaner, context.getLocationFactory(), TimeUnit.DAYS.toMillis(fileRetentionDurationDays), fileCleanupBatchSize);
            scheduledExecutorService.scheduleAtFixedRate(logCleaner, 10, logCleanupIntervalMins, TimeUnit.MINUTES);
        }
    } else if (!Boolean.TRUE.equals(context.getObject(Constants.Logging.PIPELINE_VALIDATION))) {
        throw new IllegalStateException("Expected logger context instance of " + AppenderContext.class.getName() + " but get " + context.getClass().getName());
    }
    super.start();
}

Example 12 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by caskdata.

the class CDAPLogAppenderTest method testCDAPLogAppenderRotation.

@Test
public void testCDAPLogAppenderRotation() throws Exception {
    int syncInterval = 1024 * 1024;
    FileMetaDataReader fileMetaDataReader = injector.getInstance(FileMetaDataReader.class);
    CDAPLogAppender cdapLogAppender = new CDAPLogAppender();
    AppenderContext context = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), new NoOpMetricsCollectionService());
    context.start();
    cdapLogAppender.setSyncIntervalBytes(syncInterval);
    cdapLogAppender.setMaxFileLifetimeMs(500);
    cdapLogAppender.setMaxFileSizeInBytes(104857600);
    cdapLogAppender.setDirPermissions("750");
    cdapLogAppender.setFilePermissions("640");
    cdapLogAppender.setFileRetentionDurationDays(1);
    cdapLogAppender.setLogCleanupIntervalMins(10);
    cdapLogAppender.setFileCleanupBatchSize(100);
    cdapLogAppender.setContext(context);
    cdapLogAppender.start();
    Map<String, String> properties = new HashMap<>();
    properties.put(NamespaceLoggingContext.TAG_NAMESPACE_ID, "testTimeRotation");
    properties.put(ApplicationLoggingContext.TAG_APPLICATION_ID, "testApp");
    properties.put(UserServiceLoggingContext.TAG_USER_SERVICE_ID, "testService");
    long currentTimeMillisEvent1 = System.currentTimeMillis();
    LoggingEvent event1 = getLoggingEvent("io.cdap.Test1", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message 1", properties);
    event1.setTimeStamp(currentTimeMillisEvent1);
    cdapLogAppender.doAppend(event1);
    // Pause pass the max file lifetime ms
    TimeUnit.MILLISECONDS.sleep(500);
    long currentTimeMillisEvent2 = System.currentTimeMillis();
    LoggingEvent event2 = getLoggingEvent("io.cdap.Test2", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message 2", properties);
    event2.setTimeStamp(currentTimeMillisEvent1 + 1000);
    cdapLogAppender.doAppend(event2);
    cdapLogAppender.stop();
    context.stop();
    try {
        List<LogLocation> files = fileMetaDataReader.listFiles(cdapLogAppender.getLoggingPath(properties), 0, Long.MAX_VALUE);
        Assert.assertEquals(2, files.size());
        assertLogEventDetails(event1, files.get(0));
        assertLogEventDetails(event2, files.get(1));
        Assert.assertEquals(currentTimeMillisEvent1, files.get(0).getEventTimeMs());
        Assert.assertEquals(currentTimeMillisEvent1 + 1000, files.get(1).getEventTimeMs());
        Assert.assertTrue(files.get(0).getFileCreationTimeMs() >= currentTimeMillisEvent1);
        Assert.assertTrue(files.get(1).getFileCreationTimeMs() >= currentTimeMillisEvent2);
        // checking permission
        String expectedPermissions = "rw-r-----";
        for (LogLocation file : files) {
            Location location = file.getLocation();
            Assert.assertEquals(expectedPermissions, location.getPermissions());
        }
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 13 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by caskdata.

the class CDAPLogAppenderTest method testCDAPLogAppenderSizeBasedRotation.

@Test
public void testCDAPLogAppenderSizeBasedRotation() throws Exception {
    int syncInterval = 1024 * 1024;
    FileMetaDataReader fileMetaDataReader = injector.getInstance(FileMetaDataReader.class);
    CDAPLogAppender cdapLogAppender = new CDAPLogAppender();
    AppenderContext context = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), new NoOpMetricsCollectionService());
    context.start();
    cdapLogAppender.setSyncIntervalBytes(syncInterval);
    cdapLogAppender.setMaxFileLifetimeMs(TimeUnit.DAYS.toMillis(1));
    cdapLogAppender.setMaxFileSizeInBytes(500);
    cdapLogAppender.setDirPermissions("750");
    cdapLogAppender.setFilePermissions("640");
    cdapLogAppender.setFileRetentionDurationDays(1);
    cdapLogAppender.setLogCleanupIntervalMins(10);
    cdapLogAppender.setFileCleanupBatchSize(100);
    cdapLogAppender.setContext(context);
    cdapLogAppender.start();
    Map<String, String> properties = new HashMap<>();
    properties.put(NamespaceLoggingContext.TAG_NAMESPACE_ID, "testSizeRotation");
    properties.put(ApplicationLoggingContext.TAG_APPLICATION_ID, "testApp");
    properties.put(UserServiceLoggingContext.TAG_USER_SERVICE_ID, "testService");
    long currentTimeMillisEvent1 = System.currentTimeMillis();
    LoggingEvent event1 = getLoggingEvent("io.cdap.Test1", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message 1", properties);
    event1.setTimeStamp(currentTimeMillisEvent1);
    cdapLogAppender.doAppend(event1);
    // sync updates the file size
    cdapLogAppender.sync();
    long currentTimeMillisEvent2 = System.currentTimeMillis();
    LoggingEvent event2 = getLoggingEvent("io.cdap.Test2", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message 2", properties);
    event2.setTimeStamp(currentTimeMillisEvent2);
    // one new append, we will rotate to new file as the file size limit is very low and last append exceeded that.
    cdapLogAppender.doAppend(event2);
    cdapLogAppender.stop();
    context.stop();
    try {
        List<LogLocation> files = fileMetaDataReader.listFiles(cdapLogAppender.getLoggingPath(properties), 0, Long.MAX_VALUE);
        Assert.assertEquals(2, files.size());
        assertLogEventDetails(event1, files.get(0));
        assertLogEventDetails(event2, files.get(1));
        Assert.assertEquals(currentTimeMillisEvent1, files.get(0).getEventTimeMs());
        Assert.assertEquals(currentTimeMillisEvent2, files.get(1).getEventTimeMs());
        Assert.assertTrue(files.get(0).getFileCreationTimeMs() >= currentTimeMillisEvent1);
        Assert.assertTrue(files.get(1).getFileCreationTimeMs() >= currentTimeMillisEvent2);
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 14 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by caskdata.

the class LogBufferService method loadLogPipelines.

/**
 * Load log buffer pipelines.
 */
@SuppressWarnings("unchecked")
private List<LogBufferProcessorPipeline> loadLogPipelines() {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    List<LogBufferProcessorPipeline> bufferPipelines = new ArrayList<>();
    // Create one LogBufferProcessorPipeline per spec
    for (LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        CConfiguration cConf = pipelineSpec.getConf();
        AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf);
        LogBufferPipelineConfig config = new LogBufferPipelineConfig(bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS), cConf.getLong(Constants.LogBuffer.LOG_BUFFER_PIPELINE_BATCH_SIZE, 1000));
        CheckpointManager checkpointManager = checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix(), CheckpointManagerFactory.Type.LOG_BUFFER);
        LogBufferProcessorPipeline pipeline = new LogBufferProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), config, checkpointManager, 0);
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> pipeline, retryStrategy));
        bufferPipelines.add(pipeline);
        checkpointManagers.add(checkpointManager);
    }
    return bufferPipelines;
}

Example 15 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by caskdata.

the class LocalLogAppender method start.

@Override
public void start() {
    // Load and starts all configured log processing pipelines
    LogPipelineLoader pipelineLoader = new LogPipelineLoader(cConf);
    Map<String, LogPipelineSpecification<AppenderContext>> specs = pipelineLoader.load(() -> new LocalAppenderContext(transactionRunner, locationFactory, metricsCollectionService));
    // Use the event delay as the sync interval
    long syncIntervalMillis = cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS);
    List<LocalLogProcessorPipeline> pipelines = new ArrayList<>();
    Set<Thread> pipelineThreads = Collections.newSetFromMap(new IdentityHashMap<>());
    for (LogPipelineSpecification<AppenderContext> spec : specs.values()) {
        LogProcessorPipelineContext context = new LogProcessorPipelineContext(cConf, spec.getName(), spec.getContext(), spec.getContext().getMetricsContext(), spec.getContext().getInstanceId());
        LocalLogProcessorPipeline pipeline = new LocalLogProcessorPipeline(context, syncIntervalMillis);
        pipeline.startAndWait();
        pipelineThreads.add(pipeline.getAppenderThread());
        pipelines.add(pipeline);
    }
    this.pipelines.getAndSet(pipelines).forEach(LocalLogProcessorPipeline::stopAndWait);
    this.pipelineThreads.set(pipelineThreads);
    super.start();
}