Examples with MapReduceLoggingContext io.cdap.cdap.logging.context.MapReduceLoggingContext used on opensource projects

Example 1 with MapReduceLoggingContext

use of io.cdap.cdap.logging.context.MapReduceLoggingContext in project cdap by caskdata.

the class TestTMSLogging method testTmsLogAppender.

@Test
public void testTmsLogAppender() throws Exception {
    // setup TMSLogAppender and log messages to it
    LogAppenderInitializer logAppenderInitializer = new LogAppenderInitializer(tmsLogAppender);
    logAppenderInitializer.initialize("TestTMSLogging");
    Logger logger = LoggerFactory.getLogger("TestTMSLogging");
    LoggingTester loggingTester = new LoggingTester();
    LoggingContext loggingContext = new MapReduceLoggingContext("TKL_NS_1", "APP_1", "MR_1", "RUN1");
    loggingTester.generateLogs(logger, loggingContext);
    logAppenderInitializer.close();
    // fetch and deserialize all the logs from TMS
    LoggingEventSerializer loggingEventSerializer = new LoggingEventSerializer();
    Map<Integer, List<ILoggingEvent>> partitionedFetchedLogs = new HashMap<>();
    int totalFetchedLogs = 0;
    for (Map.Entry<Integer, TopicId> topicId : topicIds.entrySet()) {
        List<ILoggingEvent> fetchedLogs = new ArrayList<>();
        MessageFetcher messageFetcher = client.prepareFetch(topicId.getValue());
        try (CloseableIterator<RawMessage> messages = messageFetcher.fetch()) {
            while (messages.hasNext()) {
                RawMessage message = messages.next();
                ILoggingEvent iLoggingEvent = loggingEventSerializer.fromBytes(ByteBuffer.wrap(message.getPayload()));
                fetchedLogs.add(iLoggingEvent);
            }
        }
        totalFetchedLogs += fetchedLogs.size();
        partitionedFetchedLogs.put(topicId.getKey(), fetchedLogs);
    }
    // LoggingTester emits 240 logs in total
    Assert.assertEquals(240, totalFetchedLogs);
    // Read the partition that our LoggingContext maps to and filter the logs in there to the logs that correspond
    // to our LoggingContext.
    LogPartitionType logPartitionType = LogPartitionType.valueOf(cConf.get(Constants.Logging.LOG_PUBLISH_PARTITION_KEY).toUpperCase());
    String partitionKey = logPartitionType.getPartitionKey(loggingContext);
    int partition = TMSLogAppender.partition(partitionKey, cConf.getInt(Constants.Logging.NUM_PARTITIONS));
    Filter logFilter = LoggingContextHelper.createFilter(loggingContext);
    List<ILoggingEvent> filteredLogs = partitionedFetchedLogs.get(partition).stream().filter(logFilter::match).collect(Collectors.toList());
    // LoggingTester emits 60 logs with the given LoggingContext
    Assert.assertEquals(60, filteredLogs.size());
    for (int i = 0; i < filteredLogs.size(); i++) {
        ILoggingEvent loggingEvent = filteredLogs.get(i);
        Assert.assertEquals(String.format("Test log message %s arg1 arg2", i), loggingEvent.getFormattedMessage());
    }
}

Example 2 with MapReduceLoggingContext

use of io.cdap.cdap.logging.context.MapReduceLoggingContext in project cdap by caskdata.

the class MockLogReader method generateLogs.

public void generateLogs() throws InterruptedException {
    // Add logs for app testApp2, worker testWorker1
    generateLogs(new WorkerLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testApp2", "testWorker1", "", ""), NamespaceId.DEFAULT.app("testApp2").worker("testWorker1"), ProgramRunStatus.RUNNING);
    // Add logs for app testApp3, mapreduce testMapReduce1
    generateLogs(new MapReduceLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testApp3", "testMapReduce1", ""), NamespaceId.DEFAULT.app("testApp3").mr("testMapReduce1"), ProgramRunStatus.SUSPENDED);
    // Add logs for app testApp1, service testService1
    generateLogs(new UserServiceLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testApp4", "testService1", "test1", "", ""), NamespaceId.DEFAULT.app("testApp4").service("testService1"), ProgramRunStatus.RUNNING);
    // Add logs for app testApp1, mapreduce testMapReduce1
    generateLogs(new MapReduceLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testTemplate1", "testMapReduce1", ""), TEST_NAMESPACE_ID.app("testTemplate1").mr("testMapReduce1"), ProgramRunStatus.COMPLETED);
    // Add logs for app testApp1, worker testWorker1 in testNamespace
    generateLogs(new WorkerLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testApp1", "testWorker1", "", ""), TEST_NAMESPACE_ID.app("testApp1").worker("testWorker1"), ProgramRunStatus.COMPLETED);
    // Add logs for app testApp1, service testService1 in testNamespace
    generateLogs(new UserServiceLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testApp4", "testService1", "test1", "", ""), TEST_NAMESPACE_ID.app("testApp4").service("testService1"), ProgramRunStatus.KILLED);
    // Add logs for testWorkflow1 in testNamespace
    generateLogs(new WorkflowLoggingContext(TEST_NAMESPACE_ID.getNamespace(), "testTemplate1", "testWorkflow1", "testRun1"), TEST_NAMESPACE_ID.app("testTemplate1").workflow("testWorkflow1"), ProgramRunStatus.COMPLETED);
    // Add logs for testWorkflow1 in default namespace
    generateLogs(new WorkflowLoggingContext(NamespaceId.DEFAULT.getEntityName(), "testTemplate1", "testWorkflow1", "testRun2"), NamespaceId.DEFAULT.app("testTemplate1").workflow("testWorkflow1"), ProgramRunStatus.COMPLETED);
    generateWorkflowLogs();
}