use of org.graylog.integrations.aws.cloudwatch.KinesisLogEntry in project graylog-plugin-integrations by Graylog2.
the class KinesisService method detectAndParseMessage.
/**
* Detect the message type.
*
* @param logMessage A string containing the actual log message.
* @param timestamp The message timestamp.
* @param kinesisStreamName The stream name.
* @param logGroupName The CloudWatch log group name.
* @param logStreamName The CloudWatch log stream name.
* @param compressed Indicates if the payload is compressed and probably from CloudWatch.
* @return A {@code KinesisHealthCheckResponse} with the fully parsed message and type.
*/
private KinesisHealthCheckResponse detectAndParseMessage(String logMessage, DateTime timestamp, String kinesisStreamName, String logGroupName, String logStreamName, boolean compressed) {
LOG.debug("Attempting to detect the type of log message. message [{}] stream [{}] log group [{}].", logMessage, kinesisStreamName, logGroupName);
final AWSLogMessage awsLogMessage = new AWSLogMessage(logMessage);
AWSMessageType awsMessageType = awsLogMessage.detectLogMessageType(compressed);
LOG.debug("The message is type [{}]", awsMessageType);
final String responseMessage = String.format("Success. The message is a %s message.", awsMessageType.getLabel());
final KinesisLogEntry logEvent = KinesisLogEntry.create(kinesisStreamName, logGroupName, logStreamName, timestamp, logMessage);
final Codec.Factory<? extends Codec> codecFactory = this.availableCodecs.get(awsMessageType.getCodecName());
if (codecFactory == null) {
throw new BadRequestException(String.format("A codec with name [%s] could not be found.", awsMessageType.getCodecName()));
}
// TODO: Do we need to provide a valid configuration here?
final Codec codec = codecFactory.create(Configuration.EMPTY_CONFIGURATION);
final byte[] payload;
try {
payload = objectMapper.writeValueAsBytes(logEvent);
} catch (JsonProcessingException e) {
throw new BadRequestException("Encoding the message to bytes failed.", e);
}
final Message fullyParsedMessage = codec.decode(new RawMessage(payload));
if (fullyParsedMessage == null) {
throw new BadRequestException(String.format("Message decoding failed. More information might be " + "available by enabling Debug logging. message [%s]", logMessage));
}
LOG.debug("Successfully parsed message type [{}] with codec [{}].", awsMessageType, awsMessageType.getCodecName());
return KinesisHealthCheckResponse.create(awsMessageType, responseMessage, fullyParsedMessage.getFields());
}
use of org.graylog.integrations.aws.cloudwatch.KinesisLogEntry in project graylog-plugin-integrations by Graylog2.
the class AWSCodecTest method testKinesisRawCodec.
@Test
public void testKinesisRawCodec() throws JsonProcessingException {
final HashMap<String, Object> configMap = new HashMap<>();
configMap.put(AWSCodec.CK_AWS_MESSAGE_TYPE, AWSMessageType.KINESIS_RAW.toString());
final Configuration configuration = new Configuration(configMap);
final AWSCodec codec = new AWSCodec(configuration, AWSTestingUtils.buildTestCodecs());
final DateTime timestamp = DateTime.now(DateTimeZone.UTC);
final KinesisLogEntry kinesisLogEntry = KinesisLogEntry.create("a-stream", "log-group", "log-stream", timestamp, "This a raw message");
Message message = codec.decode(new RawMessage(objectMapper.writeValueAsBytes(kinesisLogEntry)));
Assert.assertEquals("log-group", message.getField(AbstractKinesisCodec.FIELD_LOG_GROUP));
Assert.assertEquals("log-stream", message.getField(AbstractKinesisCodec.FIELD_LOG_STREAM));
Assert.assertEquals("a-stream", message.getField(AbstractKinesisCodec.FIELD_KINESIS_STREAM));
Assert.assertEquals(KinesisRawLogCodec.SOURCE, message.getField("source"));
Assert.assertEquals("This a raw message", message.getField("message"));
Assert.assertEquals(timestamp, message.getTimestamp());
}
use of org.graylog.integrations.aws.cloudwatch.KinesisLogEntry in project graylog-plugin-integrations by Graylog2.
the class CloudWatchFlowLogCodecTest method testFlowLogCodecValues.
/**
* Verify that the correct values are parsed by the Flow Log codec.
*/
@Test
public void testFlowLogCodecValues() {
final String flowLogMessage = "2 423432432432 eni-3244234 172.1.1.2 172.1.1.2 80 2264 6 1 52 1559738144 1559738204 ACCEPT OK";
final DateTime timestamp = DateTime.now(DateTimeZone.UTC);
final KinesisLogEntry logEvent = KinesisLogEntry.create("a-stream", "log-group", "log-stream", timestamp, flowLogMessage);
final Message message = codec.decodeLogData(logEvent);
Assert.assertEquals("log-group", message.getField(AbstractKinesisCodec.FIELD_LOG_GROUP));
Assert.assertEquals("log-stream", message.getField(AbstractKinesisCodec.FIELD_LOG_STREAM));
Assert.assertEquals("a-stream", message.getField(AbstractKinesisCodec.FIELD_KINESIS_STREAM));
Assert.assertEquals(6, message.getField(KinesisCloudWatchFlowLogCodec.FIELD_PROTOCOL_NUMBER));
Assert.assertEquals("172.1.1.2", message.getField(KinesisCloudWatchFlowLogCodec.FIELD_SRC_ADDR));
Assert.assertEquals(KinesisCloudWatchFlowLogCodec.SOURCE, message.getField("source"));
Assert.assertEquals("eni-3244234 ACCEPT TCP 172.1.1.2:80 -> 172.1.1.2:2264", message.getField("message"));
Assert.assertEquals(1L, message.getField(KinesisCloudWatchFlowLogCodec.FIELD_PACKETS));
Assert.assertEquals(80, message.getField(KinesisCloudWatchFlowLogCodec.FIELD_SRC_PORT));
Assert.assertEquals(60, message.getField(KinesisCloudWatchFlowLogCodec.FIELD_CAPTURE_WINDOW_DURATION));
Assert.assertEquals("TCP", message.getField(KinesisCloudWatchFlowLogCodec.FIELD_PROTOCOL));
Assert.assertEquals("423432432432", message.getField(KinesisCloudWatchFlowLogCodec.FIELD_ACCOUNT_ID));
Assert.assertEquals("eni-3244234", message.getField(KinesisCloudWatchFlowLogCodec.FIELD_INTERFACE_ID));
Assert.assertEquals("OK", message.getField(KinesisCloudWatchFlowLogCodec.FIELD_LOG_STATUS));
Assert.assertEquals(52L, message.getField(KinesisCloudWatchFlowLogCodec.FIELD_BYTES));
Assert.assertEquals(true, message.getField(KinesisCloudWatchFlowLogCodec.SOURCE_GROUP_IDENTIFIER));
Assert.assertEquals("172.1.1.2", message.getField(KinesisCloudWatchFlowLogCodec.FIELD_DST_ADDR));
Assert.assertEquals(2264, message.getField(KinesisCloudWatchFlowLogCodec.FIELD_DST_PORT));
Assert.assertEquals("ACCEPT", message.getField(KinesisCloudWatchFlowLogCodec.FIELD_ACTION));
Assert.assertEquals(timestamp, message.getTimestamp());
}
use of org.graylog.integrations.aws.cloudwatch.KinesisLogEntry in project graylog-plugin-integrations by Graylog2.
the class KinesisPayloadDecoderTest method testCloudWatchRawDecoding.
@Test
public void testCloudWatchRawDecoding() throws IOException {
final List<KinesisLogEntry> logEntries = flowLogDecoder.processMessages(AWSTestingUtils.cloudWatchRawPayload(), Instant.now());
Assert.assertEquals(2, logEntries.size());
// Verify that there are two flow logs present in the parsed result.
Assert.assertEquals(2, logEntries.stream().filter(logEntry -> {
final AWSLogMessage logMessage = new AWSLogMessage(logEntry.message());
return logMessage.detectLogMessageType(true) == AWSMessageType.KINESIS_CLOUDWATCH_RAW;
}).count());
// Verify that both messages have to correct timestamp.
Assert.assertEquals(2, logEntries.stream().filter(logEntry -> logEntry.timestamp().equals(AWSTestingUtils.CLOUD_WATCH_TIMESTAMP)).count());
}
use of org.graylog.integrations.aws.cloudwatch.KinesisLogEntry in project graylog-plugin-integrations by Graylog2.
the class KinesisPayloadDecoderTest method testCloudWatchFlowLogDecoding.
@Test
public void testCloudWatchFlowLogDecoding() throws IOException {
final List<KinesisLogEntry> logEntries = flowLogDecoder.processMessages(AWSTestingUtils.cloudWatchFlowLogPayload(), Instant.ofEpochMilli(AWSTestingUtils.CLOUD_WATCH_TIMESTAMP.getMillis()));
Assert.assertEquals(2, logEntries.size());
// Verify that there are two flowlogs present in the parsed result.
Assert.assertEquals(2, logEntries.stream().filter(logEntry -> {
final AWSLogMessage logMessage = new AWSLogMessage(logEntry.message());
return logMessage.isFlowLog();
}).count());
// Verify that both messages have to correct timestamp.
Assert.assertEquals(2, logEntries.stream().filter(logEntry -> logEntry.timestamp().equals(AWSTestingUtils.CLOUD_WATCH_TIMESTAMP)).count());
}
Aggregations