use of co.cask.cdap.logging.pipeline.LogPipelineConfigurator in project cdap by caskdata.
the class KafkaLogProcessorPipelineTest method testMetricsAppender.
@Test
public void testMetricsAppender() throws Exception {
Injector injector = KAFKA_TESTER.getInjector();
MetricsCollectionService collectionService = injector.getInstance(MetricsCollectionService.class);
collectionService.startAndWait();
LoggerContext loggerContext = new LocalAppenderContext(injector.getInstance(DatasetFramework.class), injector.getInstance(TransactionSystemClient.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetricsCollectionService.class));
final File logDir = TEMP_FOLDER.newFolder();
loggerContext.putProperty("logDirectory", logDir.getAbsolutePath());
LogPipelineConfigurator configurator = new LogPipelineConfigurator(CConfiguration.create());
configurator.setContext(loggerContext);
URL configURL = getClass().getClassLoader().getResource("pipeline-metric-appender.xml");
Assert.assertNotNull(configURL);
configurator.doConfigure(configURL);
String topic = "metricsPipeline";
TestCheckpointManager checkpointManager = new TestCheckpointManager();
KafkaPipelineConfig config = new KafkaPipelineConfig(topic, Collections.singleton(0), 1024L, 100L, 1048576, 200L);
KAFKA_TESTER.createTopic(topic, 1);
loggerContext.start();
KafkaLogProcessorPipeline pipeline = new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "testMetricAppender", loggerContext, NO_OP_METRICS_CONTEXT, 0), checkpointManager, KAFKA_TESTER.getBrokerService(), config);
pipeline.startAndWait();
// Publish some log messages to Kafka
long now = System.currentTimeMillis();
FlowletLoggingContext flowletLoggingContext = new FlowletLoggingContext("default", "app1", "flow1", "flowlet1", "run1", "instance1");
publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.INFO, "0", now - 1000), createLoggingEvent("test.logger", Level.INFO, "2", now - 700), createLoggingEvent("test.logger", Level.INFO, "3", now - 500), createLoggingEvent("test.logger", Level.INFO, "1", now - 900), createLoggingEvent("test.logger", Level.DEBUG, "hidden", now - 600), createLoggingEvent("test.logger", Level.INFO, "4", now - 100)), flowletLoggingContext);
WorkflowProgramLoggingContext workflowProgramLoggingContext = new WorkflowProgramLoggingContext("default", "app1", "wflow1", "run1", ProgramType.MAPREDUCE, "mr1", "mrun1");
publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.WARN, "0", now - 1000), createLoggingEvent("test.logger", Level.WARN, "2", now - 700), createLoggingEvent("test.logger", Level.TRACE, "3", now - 500)), workflowProgramLoggingContext);
ServiceLoggingContext serviceLoggingContext = new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.TRANSACTION);
publishLog(topic, ImmutableList.of(createLoggingEvent("test.logger", Level.ERROR, "0", now - 1000), createLoggingEvent("test.logger", Level.ERROR, "2", now - 700), createLoggingEvent("test.logger", Level.ERROR, "3", now - 500), createLoggingEvent("test.logger", Level.INFO, "1", now - 900)), serviceLoggingContext);
final MetricStore metricStore = injector.getInstance(MetricStore.class);
try {
verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.info", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(flowletLoggingContext), new ArrayList<String>()), 5L);
verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.debug", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(flowletLoggingContext), new ArrayList<String>()), 1L);
verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.warn", AggregationFunction.SUM, // mapreduce metrics context
ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.MAPREDUCE, "mr1", Constants.Metrics.Tag.RUN_ID, "mrun1"), new ArrayList<String>()), 2L);
verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.app.log.trace", AggregationFunction.SUM, // workflow metrics context
ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, "default", Constants.Metrics.Tag.APP, "app1", Constants.Metrics.Tag.WORKFLOW, "wflow1", Constants.Metrics.Tag.RUN_ID, "run1"), new ArrayList<String>()), 1L);
verifyMetricsWithRetry(metricStore, new MetricDataQuery(0, Integer.MAX_VALUE, Integer.MAX_VALUE, "system.services.log.error", AggregationFunction.SUM, LoggingContextHelper.getMetricsTags(serviceLoggingContext), new ArrayList<String>()), 3L);
} finally {
pipeline.stopAndWait();
loggerContext.stop();
collectionService.stopAndWait();
}
}
use of co.cask.cdap.logging.pipeline.LogPipelineConfigurator in project cdap by caskdata.
the class KafkaLogProcessorPipelineTest method testMultiAppenders.
@Test
public void testMultiAppenders() throws Exception {
final File logDir = TEMP_FOLDER.newFolder();
LoggerContext loggerContext = new LoggerContext();
loggerContext.putProperty("logDirectory", logDir.getAbsolutePath());
LogPipelineConfigurator configurator = new LogPipelineConfigurator(CConfiguration.create());
configurator.setContext(loggerContext);
URL configURL = getClass().getClassLoader().getResource("pipeline-multi-appenders.xml");
Assert.assertNotNull(configURL);
configurator.doConfigure(configURL);
String topic = "testMultiAppenders";
TestCheckpointManager checkpointManager = new TestCheckpointManager();
KafkaPipelineConfig config = new KafkaPipelineConfig(topic, Collections.singleton(0), 1024L, 100L, 1048576, 200L);
KAFKA_TESTER.createTopic(topic, 1);
loggerContext.start();
KafkaLogProcessorPipeline pipeline = new KafkaLogProcessorPipeline(new LogProcessorPipelineContext(CConfiguration.create(), "testMultiAppenders", loggerContext, NO_OP_METRICS_CONTEXT, 0), checkpointManager, KAFKA_TESTER.getBrokerService(), config);
pipeline.startAndWait();
// Publish some log messages to Kafka using a non-specific logger
long now = System.currentTimeMillis();
publishLog(topic, ImmutableList.of(createLoggingEvent("logger.trace", Level.TRACE, "TRACE", now - 1000), createLoggingEvent("logger.debug", Level.DEBUG, "DEBUG", now - 900), createLoggingEvent("logger.info", Level.INFO, "INFO", now - 800), createLoggingEvent("logger.warn", Level.WARN, "WARN", now - 700), createLoggingEvent("logger.error", Level.ERROR, "ERROR", now - 600)));
// All logs should get logged to the default.log file
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
File logFile = new File(logDir, "default.log");
List<String> lines = Files.readAllLines(logFile.toPath(), StandardCharsets.UTF_8);
return Arrays.asList("TRACE", "DEBUG", "INFO", "WARN", "ERROR").equals(lines);
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Publish some more log messages via the non-additive "test.info" logger.
now = System.currentTimeMillis();
publishLog(topic, ImmutableList.of(createLoggingEvent("test.info.trace", Level.TRACE, "TRACE", now - 1000), createLoggingEvent("test.info.debug", Level.DEBUG, "DEBUG", now - 900), createLoggingEvent("test.info", Level.INFO, "INFO", now - 800), createLoggingEvent("test.info.warn", Level.WARN, "WARN", now - 700), createLoggingEvent("test.info.error", Level.ERROR, "ERROR", now - 600)));
// Only logs with INFO or above level should get written to the info.log file
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
File logFile = new File(logDir, "info.log");
List<String> lines = Files.readAllLines(logFile.toPath(), StandardCharsets.UTF_8);
return Arrays.asList("INFO", "WARN", "ERROR").equals(lines);
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// The default.log file shouldn't be changed, because the test.info logger is non additive
File defaultLogFile = new File(logDir, "default.log");
List<String> lines = Files.readAllLines(defaultLogFile.toPath(), StandardCharsets.UTF_8);
Assert.assertEquals(Arrays.asList("TRACE", "DEBUG", "INFO", "WARN", "ERROR"), lines);
// Publish a log messages via the additive "test.error" logger.
now = System.currentTimeMillis();
publishLog(topic, ImmutableList.of(createLoggingEvent("test.error.1.2", Level.ERROR, "ERROR", now - 1000)));
// Expect the log get appended to both the error.log file as well as the default.log file
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
File logFile = new File(logDir, "error.log");
List<String> lines = Files.readAllLines(logFile.toPath(), StandardCharsets.UTF_8);
if (!Collections.singletonList("ERROR").equals(lines)) {
return false;
}
logFile = new File(logDir, "default.log");
lines = Files.readAllLines(logFile.toPath(), StandardCharsets.UTF_8);
return Arrays.asList("TRACE", "DEBUG", "INFO", "WARN", "ERROR", "ERROR").equals(lines);
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
pipeline.stopAndWait();
loggerContext.stop();
}
use of co.cask.cdap.logging.pipeline.LogPipelineConfigurator in project cdap by caskdata.
the class LogPipelineLoader method load.
/**
* Creates the {@link LogPipelineSpecification} representing the given log pipeline configuration URL.
*/
private <T extends LoggerContext> LogPipelineSpecification<T> load(Provider<T> contextProvider, URL configURL) throws JoranException {
// Create one AppenderContext per config file
T context = contextProvider.get();
CConfiguration pipelineCConf = CConfiguration.copy(cConf);
LogPipelineConfigurator configurator = new LogPipelineConfigurator(pipelineCConf);
configurator.setContext(context);
configurator.doConfigure(configURL);
// Check if the configuration has any error in it.
if (!new StatusChecker(context).isErrorFree(Status.ERROR)) {
List<Status> errors = new ArrayList<>();
for (Status status : context.getStatusManager().getCopyOfStatusList()) {
if (status.getEffectiveLevel() == Status.ERROR) {
errors.add(status);
}
}
throw new JoranException("Configuration failed " + errors);
}
// Default the pipeline name to the config file name (without extension) if it is not set
if (context.getName() == null) {
String path = configURL.getPath();
int idx = path.lastIndexOf("/");
int dotIdx = path.lastIndexOf('.');
int startIdx = idx < 0 ? 0 : idx + 1;
int endIdx = dotIdx > idx ? dotIdx : path.length();
context.setName(path.substring(startIdx, endIdx));
}
byte[] checkpointPrefix = Bytes.toBytes(context.getName());
String prefixNum = configurator.getExecutionContext().getProperty(Constants.Logging.PIPELINE_CHECKPOINT_PREFIX_NUM);
if (prefixNum != null) {
try {
checkpointPrefix = Bytes.toBytes(Integer.parseInt(prefixNum));
} catch (NumberFormatException e) {
LOG.warn("Ignoring invalid {} setting for pipeline in {}", Constants.Logging.PIPELINE_CHECKPOINT_PREFIX_NUM, configURL);
}
}
return new LogPipelineSpecification<>(configURL, context, setupPipelineCConf(configurator, pipelineCConf), checkpointPrefix);
}
use of co.cask.cdap.logging.pipeline.LogPipelineConfigurator in project cdap by caskdata.
the class KafkaLogProcessorPipelineTest method createLoggerContext.
private LoggerContext createLoggerContext(String rootLevel, Map<String, String> loggerLevels, String appenderClassName) throws Exception {
Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
Element configuration = doc.createElement("configuration");
doc.appendChild(configuration);
Element appender = doc.createElement("appender");
appender.setAttribute("name", "Test");
appender.setAttribute("class", appenderClassName);
configuration.appendChild(appender);
for (Map.Entry<String, String> entry : loggerLevels.entrySet()) {
Element logger = doc.createElement("logger");
logger.setAttribute("name", entry.getKey());
logger.setAttribute("level", entry.getValue());
configuration.appendChild(logger);
}
Element rootLogger = doc.createElement("root");
rootLogger.setAttribute("level", rootLevel);
Element appenderRef = doc.createElement("appender-ref");
appenderRef.setAttribute("ref", "Test");
rootLogger.appendChild(appenderRef);
configuration.appendChild(rootLogger);
Transformer transformer = TransformerFactory.newInstance().newTransformer();
StringWriter writer = new StringWriter();
transformer.transform(new DOMSource(doc), new StreamResult(writer));
LoggerContext context = new LoggerContext();
JoranConfigurator configurator = new LogPipelineConfigurator(CConfiguration.create());
configurator.setContext(context);
configurator.doConfigure(new InputSource(new StringReader(writer.toString())));
return context;
}
Aggregations