Search in sources :

Example 6 with LoggingEventSerializer

use of io.cdap.cdap.logging.serialize.LoggingEventSerializer in project cdap by cdapio.

the class KafkaOffsetResolverTest method publishLog.

private void publishLog(String topic, Iterable<ILoggingEvent> events, LoggingContext context) {
    KafkaPublisher.Preparer preparer = KAFKA_TESTER.getKafkaClient().getPublisher(KafkaPublisher.Ack.LEADER_RECEIVED, Compression.NONE).prepare(topic);
    LoggingEventSerializer serializer = new LoggingEventSerializer();
    for (ILoggingEvent event : events) {
        preparer.add(ByteBuffer.wrap(serializer.toBytes(event)), context.getLogPartition());
    }
    preparer.send();
}
Also used : LoggingEventSerializer(io.cdap.cdap.logging.serialize.LoggingEventSerializer) ILoggingEvent(ch.qos.logback.classic.spi.ILoggingEvent) KafkaPublisher(org.apache.twill.kafka.client.KafkaPublisher)

Example 7 with LoggingEventSerializer

use of io.cdap.cdap.logging.serialize.LoggingEventSerializer in project cdap by cdapio.

the class TestKafkaLogging method convertFetchedMessage.

private Map.Entry<Integer, String> convertFetchedMessage(FetchedMessage message) throws IOException {
    LoggingEventSerializer serializer = new LoggingEventSerializer();
    ILoggingEvent iLoggingEvent = serializer.fromBytes(message.getPayload());
    LoggingContext loggingContext = LoggingContextHelper.getLoggingContext(iLoggingEvent.getMDCPropertyMap());
    String key = loggingContext.getLogPartition();
    return Maps.immutableEntry(message.getTopicPartition().getPartition(), key);
}
Also used : LoggingContext(io.cdap.cdap.common.logging.LoggingContext) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) LoggingEventSerializer(io.cdap.cdap.logging.serialize.LoggingEventSerializer) ILoggingEvent(ch.qos.logback.classic.spi.ILoggingEvent)

Example 8 with LoggingEventSerializer

use of io.cdap.cdap.logging.serialize.LoggingEventSerializer in project cdap by caskdata.

the class TestTMSLogging method testTmsLogAppender.

@Test
public void testTmsLogAppender() throws Exception {
    // setup TMSLogAppender and log messages to it
    LogAppenderInitializer logAppenderInitializer = new LogAppenderInitializer(tmsLogAppender);
    logAppenderInitializer.initialize("TestTMSLogging");
    Logger logger = LoggerFactory.getLogger("TestTMSLogging");
    LoggingTester loggingTester = new LoggingTester();
    LoggingContext loggingContext = new MapReduceLoggingContext("TKL_NS_1", "APP_1", "MR_1", "RUN1");
    loggingTester.generateLogs(logger, loggingContext);
    logAppenderInitializer.close();
    // fetch and deserialize all the logs from TMS
    LoggingEventSerializer loggingEventSerializer = new LoggingEventSerializer();
    Map<Integer, List<ILoggingEvent>> partitionedFetchedLogs = new HashMap<>();
    int totalFetchedLogs = 0;
    for (Map.Entry<Integer, TopicId> topicId : topicIds.entrySet()) {
        List<ILoggingEvent> fetchedLogs = new ArrayList<>();
        MessageFetcher messageFetcher = client.prepareFetch(topicId.getValue());
        try (CloseableIterator<RawMessage> messages = messageFetcher.fetch()) {
            while (messages.hasNext()) {
                RawMessage message = messages.next();
                ILoggingEvent iLoggingEvent = loggingEventSerializer.fromBytes(ByteBuffer.wrap(message.getPayload()));
                fetchedLogs.add(iLoggingEvent);
            }
        }
        totalFetchedLogs += fetchedLogs.size();
        partitionedFetchedLogs.put(topicId.getKey(), fetchedLogs);
    }
    // LoggingTester emits 240 logs in total
    Assert.assertEquals(240, totalFetchedLogs);
    // Read the partition that our LoggingContext maps to and filter the logs in there to the logs that correspond
    // to our LoggingContext.
    LogPartitionType logPartitionType = LogPartitionType.valueOf(cConf.get(Constants.Logging.LOG_PUBLISH_PARTITION_KEY).toUpperCase());
    String partitionKey = logPartitionType.getPartitionKey(loggingContext);
    int partition = TMSLogAppender.partition(partitionKey, cConf.getInt(Constants.Logging.NUM_PARTITIONS));
    Filter logFilter = LoggingContextHelper.createFilter(loggingContext);
    List<ILoggingEvent> filteredLogs = partitionedFetchedLogs.get(partition).stream().filter(logFilter::match).collect(Collectors.toList());
    // LoggingTester emits 60 logs with the given LoggingContext
    Assert.assertEquals(60, filteredLogs.size());
    for (int i = 0; i < filteredLogs.size(); i++) {
        ILoggingEvent loggingEvent = filteredLogs.get(i);
        Assert.assertEquals(String.format("Test log message %s arg1 arg2", i), loggingEvent.getFormattedMessage());
    }
}
Also used : MessageFetcher(io.cdap.cdap.messaging.MessageFetcher) MapReduceLoggingContext(io.cdap.cdap.logging.context.MapReduceLoggingContext) LoggingContext(io.cdap.cdap.common.logging.LoggingContext) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) LoggingEventSerializer(io.cdap.cdap.logging.serialize.LoggingEventSerializer) Logger(org.slf4j.Logger) ILoggingEvent(ch.qos.logback.classic.spi.ILoggingEvent) MapReduceLoggingContext(io.cdap.cdap.logging.context.MapReduceLoggingContext) LoggingTester(io.cdap.cdap.logging.appender.LoggingTester) LogAppenderInitializer(io.cdap.cdap.logging.appender.LogAppenderInitializer) LogPartitionType(io.cdap.cdap.logging.appender.kafka.LogPartitionType) Filter(io.cdap.cdap.logging.filter.Filter) ArrayList(java.util.ArrayList) List(java.util.List) TopicId(io.cdap.cdap.proto.id.TopicId) RawMessage(io.cdap.cdap.messaging.data.RawMessage) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.junit.Test)

Example 9 with LoggingEventSerializer

use of io.cdap.cdap.logging.serialize.LoggingEventSerializer in project cdap by caskdata.

the class TestKafkaLogging method convertFetchedMessage.

private Map.Entry<Integer, String> convertFetchedMessage(FetchedMessage message) throws IOException {
    LoggingEventSerializer serializer = new LoggingEventSerializer();
    ILoggingEvent iLoggingEvent = serializer.fromBytes(message.getPayload());
    LoggingContext loggingContext = LoggingContextHelper.getLoggingContext(iLoggingEvent.getMDCPropertyMap());
    String key = loggingContext.getLogPartition();
    return Maps.immutableEntry(message.getTopicPartition().getPartition(), key);
}
Also used : LoggingContext(io.cdap.cdap.common.logging.LoggingContext) WorkerLoggingContext(io.cdap.cdap.logging.context.WorkerLoggingContext) LoggingEventSerializer(io.cdap.cdap.logging.serialize.LoggingEventSerializer) ILoggingEvent(ch.qos.logback.classic.spi.ILoggingEvent)

Example 10 with LoggingEventSerializer

use of io.cdap.cdap.logging.serialize.LoggingEventSerializer in project cdap by cdapio.

the class LogAppenderLogProcessor method process.

@Override
public void process(Iterator<byte[]> loggingEventBytes) {
    LoggingEventSerializer serializer = LOGGING_EVENT_SERIALIZER.get();
    loggingEventBytes.forEachRemaining(bytes -> {
        try {
            ILoggingEvent iLoggingEvent = serializer.fromBytes(ByteBuffer.wrap(bytes));
            LoggingContext loggingContext = LoggingContextHelper.getLoggingContext(iLoggingEvent.getMDCPropertyMap());
            if (loggingContext == null) {
                // This shouldn't happen
                LOG.debug("Ignore logging event due to missing logging context: {}", iLoggingEvent);
                return;
            }
            logAppender.append(new LogMessage(iLoggingEvent, loggingContext));
        } catch (IOException e) {
            LOG.warn("Ignore logging event due to decode failure: {}", e.getMessage());
            LOG.debug("Ignore logging event stack trace", e);
        }
    });
}
Also used : LoggingContext(io.cdap.cdap.common.logging.LoggingContext) LogMessage(io.cdap.cdap.logging.appender.LogMessage) LoggingEventSerializer(io.cdap.cdap.logging.serialize.LoggingEventSerializer) IOException(java.io.IOException) ILoggingEvent(ch.qos.logback.classic.spi.ILoggingEvent)

Aggregations

ILoggingEvent (ch.qos.logback.classic.spi.ILoggingEvent)12 LoggingEventSerializer (io.cdap.cdap.logging.serialize.LoggingEventSerializer)12 LoggingContext (io.cdap.cdap.common.logging.LoggingContext)6 LogMessage (io.cdap.cdap.logging.appender.LogMessage)6 KafkaPublisher (org.apache.twill.kafka.client.KafkaPublisher)6 LogAppenderInitializer (io.cdap.cdap.logging.appender.LogAppenderInitializer)2 LoggingTester (io.cdap.cdap.logging.appender.LoggingTester)2 LogPartitionType (io.cdap.cdap.logging.appender.kafka.LogPartitionType)2 MapReduceLoggingContext (io.cdap.cdap.logging.context.MapReduceLoggingContext)2 WorkerLoggingContext (io.cdap.cdap.logging.context.WorkerLoggingContext)2 Filter (io.cdap.cdap.logging.filter.Filter)2 MessageFetcher (io.cdap.cdap.messaging.MessageFetcher)2 RawMessage (io.cdap.cdap.messaging.data.RawMessage)2 TopicId (io.cdap.cdap.proto.id.TopicId)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 List (java.util.List)2 Map (java.util.Map)2 Test (org.junit.Test)2