Examples with AppenderContext io.cdap.cdap.api.logging.AppenderContext used on opensource projects

Example 6 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by caskdata.

the class RollingLocationLogAppenderTest method testRollingLocationLogAppender.

@Test
public void testRollingLocationLogAppender() throws Exception {
    // assume SLF4J is bound to logback in the current environment
    AppenderContext appenderContext = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), new NoOpMetricsCollectionService());
    JoranConfigurator configurator = new JoranConfigurator();
    configurator.setContext(appenderContext);
    // Call context.reset() to clear any previous configuration, e.g. default
    // configuration. For multi-step configuration, omit calling context.reset().
    appenderContext.reset();
    configurator.doConfigure(getClass().getResourceAsStream("/rolling-appender-logback-test.xml"));
    StatusPrinter.printInCaseOfErrorsOrWarnings(appenderContext);
    RollingLocationLogAppender rollingAppender = (RollingLocationLogAppender) appenderContext.getLogger(RollingLocationLogAppenderTest.class).getAppender("rollingAppender");
    addTagsToMdc("testNamespace", "testApp");
    Logger logger = appenderContext.getLogger(RollingLocationLogAppenderTest.class);
    ingestLogs(logger, 5);
    Map<LocationIdentifier, LocationOutputStream> activeFiles = rollingAppender.getLocationManager().getActiveLocations();
    Assert.assertEquals(1, activeFiles.size());
    verifyFileOutput(activeFiles, 5);
    // different program should go to different directory
    addTagsToMdc("testNamespace", "testApp1");
    ingestLogs(logger, 5);
    activeFiles = rollingAppender.getLocationManager().getActiveLocations();
    Assert.assertEquals(2, activeFiles.size());
    verifyFileOutput(activeFiles, 5);
    // different program should go to different directory because namespace is different
    addTagsToMdc("testNamespace1", "testApp1");
    ingestLogs(logger, 5);
    activeFiles = rollingAppender.getLocationManager().getActiveLocations();
    Assert.assertEquals(3, activeFiles.size());
    verifyFileOutput(activeFiles, 5);
}

Example 7 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by cdapio.

the class RollingLocationLogAppender method start.

@Override
public void start() {
    // These should all passed. The settings are from the custom-log-pipeline.xml and
    // the context must be AppenderContext
    Preconditions.checkState(basePath != null, "Property basePath must be base directory.");
    Preconditions.checkState(filePath != null, "Property filePath must be filePath along with filename.");
    Preconditions.checkState(triggeringPolicy != null, "Property triggeringPolicy must be specified.");
    Preconditions.checkState(rollingPolicy != null, "Property rollingPolicy must be specified");
    Preconditions.checkState(encoder != null, "Property encoder must be specified.");
    Preconditions.checkState(dirPermissions != null, "Property dirPermissions cannot be null");
    Preconditions.checkState(filePermissions != null, "Property filePermissions cannot be null");
    if (context instanceof AppenderContext) {
        AppenderContext context = (AppenderContext) this.context;
        locationManager = new LocationManager(context.getLocationFactory(), basePath, dirPermissions, filePermissions, fileMaxInactiveTimeMs);
        filePath = filePath.replace("instanceId", Integer.toString(context.getInstanceId()));
    } else if (!Boolean.TRUE.equals(context.getObject(Constants.Logging.PIPELINE_VALIDATION))) {
        throw new IllegalStateException("Expected logger context instance of " + AppenderContext.class.getName() + " but got " + context.getClass().getName());
    }
    started = true;
}

Example 8 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by cdapio.

the class CDAPLogAppenderTest method testCDAPLogAppenderSizeBasedRotation.

@Test
public void testCDAPLogAppenderSizeBasedRotation() throws Exception {
    int syncInterval = 1024 * 1024;
    FileMetaDataReader fileMetaDataReader = injector.getInstance(FileMetaDataReader.class);
    CDAPLogAppender cdapLogAppender = new CDAPLogAppender();
    AppenderContext context = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), new NoOpMetricsCollectionService());
    context.start();
    cdapLogAppender.setSyncIntervalBytes(syncInterval);
    cdapLogAppender.setMaxFileLifetimeMs(TimeUnit.DAYS.toMillis(1));
    cdapLogAppender.setMaxFileSizeInBytes(500);
    cdapLogAppender.setDirPermissions("750");
    cdapLogAppender.setFilePermissions("640");
    cdapLogAppender.setFileRetentionDurationDays(1);
    cdapLogAppender.setLogCleanupIntervalMins(10);
    cdapLogAppender.setFileCleanupBatchSize(100);
    cdapLogAppender.setContext(context);
    cdapLogAppender.start();
    Map<String, String> properties = new HashMap<>();
    properties.put(NamespaceLoggingContext.TAG_NAMESPACE_ID, "testSizeRotation");
    properties.put(ApplicationLoggingContext.TAG_APPLICATION_ID, "testApp");
    properties.put(UserServiceLoggingContext.TAG_USER_SERVICE_ID, "testService");
    long currentTimeMillisEvent1 = System.currentTimeMillis();
    LoggingEvent event1 = getLoggingEvent("io.cdap.Test1", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message 1", properties);
    event1.setTimeStamp(currentTimeMillisEvent1);
    cdapLogAppender.doAppend(event1);
    // sync updates the file size
    cdapLogAppender.sync();
    long currentTimeMillisEvent2 = System.currentTimeMillis();
    LoggingEvent event2 = getLoggingEvent("io.cdap.Test2", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), Level.ERROR, "test message 2", properties);
    event2.setTimeStamp(currentTimeMillisEvent2);
    // one new append, we will rotate to new file as the file size limit is very low and last append exceeded that.
    cdapLogAppender.doAppend(event2);
    cdapLogAppender.stop();
    context.stop();
    try {
        List<LogLocation> files = fileMetaDataReader.listFiles(cdapLogAppender.getLoggingPath(properties), 0, Long.MAX_VALUE);
        Assert.assertEquals(2, files.size());
        assertLogEventDetails(event1, files.get(0));
        assertLogEventDetails(event2, files.get(1));
        Assert.assertEquals(currentTimeMillisEvent1, files.get(0).getEventTimeMs());
        Assert.assertEquals(currentTimeMillisEvent2, files.get(1).getEventTimeMs());
        Assert.assertTrue(files.get(0).getFileCreationTimeMs() >= currentTimeMillisEvent1);
        Assert.assertTrue(files.get(1).getFileCreationTimeMs() >= currentTimeMillisEvent2);
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 9 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by cdapio.

the class RollingLocationLogAppenderTest method testRollingLocationLogAppender.

@Test
public void testRollingLocationLogAppender() throws Exception {
    // assume SLF4J is bound to logback in the current environment
    AppenderContext appenderContext = new LocalAppenderContext(injector.getInstance(TransactionRunner.class), injector.getInstance(LocationFactory.class), new NoOpMetricsCollectionService());
    JoranConfigurator configurator = new JoranConfigurator();
    configurator.setContext(appenderContext);
    // Call context.reset() to clear any previous configuration, e.g. default
    // configuration. For multi-step configuration, omit calling context.reset().
    appenderContext.reset();
    configurator.doConfigure(getClass().getResourceAsStream("/rolling-appender-logback-test.xml"));
    StatusPrinter.printInCaseOfErrorsOrWarnings(appenderContext);
    RollingLocationLogAppender rollingAppender = (RollingLocationLogAppender) appenderContext.getLogger(RollingLocationLogAppenderTest.class).getAppender("rollingAppender");
    addTagsToMdc("testNamespace", "testApp");
    Logger logger = appenderContext.getLogger(RollingLocationLogAppenderTest.class);
    ingestLogs(logger, 5);
    Map<LocationIdentifier, LocationOutputStream> activeFiles = rollingAppender.getLocationManager().getActiveLocations();
    Assert.assertEquals(1, activeFiles.size());
    verifyFileOutput(activeFiles, 5);
    // different program should go to different directory
    addTagsToMdc("testNamespace", "testApp1");
    ingestLogs(logger, 5);
    activeFiles = rollingAppender.getLocationManager().getActiveLocations();
    Assert.assertEquals(2, activeFiles.size());
    verifyFileOutput(activeFiles, 5);
    // different program should go to different directory because namespace is different
    addTagsToMdc("testNamespace1", "testApp1");
    ingestLogs(logger, 5);
    activeFiles = rollingAppender.getLocationManager().getActiveLocations();
    Assert.assertEquals(3, activeFiles.size());
    verifyFileOutput(activeFiles, 5);
}

Example 10 with AppenderContext

use of io.cdap.cdap.api.logging.AppenderContext in project cdap by cdapio.

the class DistributedLogFramework method createService.

@Override
@SuppressWarnings("unchecked")
protected Service createService(Set<Integer> partitions) {
    Map<String, LogPipelineSpecification<AppenderContext>> specs = new LogPipelineLoader(cConf).load(contextProvider);
    int pipelineCount = specs.size();
    // Create one KafkaLogProcessorPipeline per spec
    final List<Service> pipelines = new ArrayList<>();
    for (final LogPipelineSpecification<AppenderContext> pipelineSpec : specs.values()) {
        final CConfiguration cConf = pipelineSpec.getConf();
        final AppenderContext context = pipelineSpec.getContext();
        long bufferSize = getBufferSize(pipelineCount, cConf, partitions.size());
        final String topic = cConf.get(Constants.Logging.KAFKA_TOPIC);
        final KafkaPipelineConfig config = new KafkaPipelineConfig(topic, partitions, bufferSize, cConf.getLong(Constants.Logging.PIPELINE_EVENT_DELAY_MS), cConf.getInt(Constants.Logging.PIPELINE_KAFKA_FETCH_SIZE), cConf.getLong(Constants.Logging.PIPELINE_CHECKPOINT_INTERVAL_MS));
        RetryStrategy retryStrategy = RetryStrategies.fromConfiguration(cConf, "system.log.process.");
        pipelines.add(new RetryOnStartFailureService(() -> new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(cConf, context.getName(), context, context.getMetricsContext(), context.getInstanceId()), checkpointManagerFactory.create(pipelineSpec.getCheckpointPrefix() + topic, CheckpointManagerFactory.Type.KAFKA), brokerService, config), retryStrategy));
    }
    // Returns a Service that start/stop all pipelines.
    return new AbstractIdleService() {

        @Override
        protected void startUp() throws Exception {
            // Starts all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::start));
        }

        @Override
        protected void shutDown() throws Exception {
            // Stops all pipeline
            validateAllFutures(Iterables.transform(pipelines, Service::stop));
        }
    };
}