Examples with AWSMessageType org.graylog.integrations.aws.AWSMessageType used on opensource projects

Example 1 with AWSMessageType

use of org.graylog.integrations.aws.AWSMessageType in project graylog-plugin-integrations by Graylog2.

the class KinesisService method detectAndParseMessage.

/**
 * Detect the message type.
 *
 * @param logMessage        A string containing the actual log message.
 * @param timestamp         The message timestamp.
 * @param kinesisStreamName The stream name.
 * @param logGroupName      The CloudWatch log group name.
 * @param logStreamName     The CloudWatch log stream name.
 * @param compressed        Indicates if the payload is compressed and probably from CloudWatch.
 * @return A {@code KinesisHealthCheckResponse} with the fully parsed message and type.
 */
private KinesisHealthCheckResponse detectAndParseMessage(String logMessage, DateTime timestamp, String kinesisStreamName, String logGroupName, String logStreamName, boolean compressed) {
    LOG.debug("Attempting to detect the type of log message. message [{}] stream [{}] log group [{}].", logMessage, kinesisStreamName, logGroupName);
    final AWSLogMessage awsLogMessage = new AWSLogMessage(logMessage);
    AWSMessageType awsMessageType = awsLogMessage.detectLogMessageType(compressed);
    LOG.debug("The message is type [{}]", awsMessageType);
    final String responseMessage = String.format("Success. The message is a %s message.", awsMessageType.getLabel());
    final KinesisLogEntry logEvent = KinesisLogEntry.create(kinesisStreamName, logGroupName, logStreamName, timestamp, logMessage);
    final Codec.Factory<? extends Codec> codecFactory = this.availableCodecs.get(awsMessageType.getCodecName());
    if (codecFactory == null) {
        throw new BadRequestException(String.format("A codec with name [%s] could not be found.", awsMessageType.getCodecName()));
    }
    // TODO: Do we need to provide a valid configuration here?
    final Codec codec = codecFactory.create(Configuration.EMPTY_CONFIGURATION);
    final byte[] payload;
    try {
        payload = objectMapper.writeValueAsBytes(logEvent);
    } catch (JsonProcessingException e) {
        throw new BadRequestException("Encoding the message to bytes failed.", e);
    }
    final Message fullyParsedMessage = codec.decode(new RawMessage(payload));
    if (fullyParsedMessage == null) {
        throw new BadRequestException(String.format("Message decoding failed. More information might be " + "available by enabling Debug logging. message [%s]", logMessage));
    }
    LOG.debug("Successfully parsed message type [{}] with codec [{}].", awsMessageType, awsMessageType.getCodecName());
    return KinesisHealthCheckResponse.create(awsMessageType, responseMessage, fullyParsedMessage.getFields());
}

Example 2 with AWSMessageType

use of org.graylog.integrations.aws.AWSMessageType in project graylog-plugin-integrations by Graylog2.

the class AWSCodec method decode.

@Nullable
@Override
public Message decode(@Nonnull RawMessage rawMessage) {
    // Load the codec by message type.
    final AWSMessageType awsMessageType = AWSMessageType.valueOf(configuration.getString(CK_AWS_MESSAGE_TYPE));
    final Codec.Factory<? extends Codec> codecFactory = this.availableCodecs.get(awsMessageType.getCodecName());
    if (codecFactory == null) {
        LOG.error("A codec with name [{}] could not be found.", awsMessageType.getCodecName());
        return null;
    }
    final Codec codec = codecFactory.create(configuration);
    // Parse the message with the specified codec.
    final Message message = codec.decode(new RawMessage(rawMessage.getPayload()));
    if (message == null) {
        LOG.error("Failed to decode message for codec [{}].", codec.getName());
        return null;
    }
    return message;
}

Example 3 with AWSMessageType

use of org.graylog.integrations.aws.AWSMessageType in project graylog-plugin-integrations by Graylog2.

the class AWSTransport method resolveTransport.

/**
 * Looks up the transport for the {@link AWSMessageType} stored in the input configuration.
 */
private Transport resolveTransport() throws MisfireException {
    final AWSMessageType awsMessageType = AWSMessageType.valueOf(configuration.getString(AWSCodec.CK_AWS_MESSAGE_TYPE));
    final Transport.Factory<? extends Transport> transportFactory = this.availableTransports.get(awsMessageType.getTransportName());
    if (transportFactory == null) {
        throw new MisfireException(String.format("A transport with name [%s] could not be found.", awsMessageType.getTransportName()));
    }
    return transportFactory.create(configuration);
}

Example 4 with AWSMessageType

use of org.graylog.integrations.aws.AWSMessageType in project graylog-plugin-integrations by Graylog2.

the class KinesisTransport method doLaunch.

@Override
public void doLaunch(MessageInput input) throws MisfireException {
    final Region region = Region.of(Objects.requireNonNull(configuration.getString(CK_AWS_REGION)));
    final String key = configuration.getString(CK_ACCESS_KEY);
    final String secret = configuration.getString(CK_SECRET_KEY);
    final String assumeRoleArn = configuration.getString(AWSInput.CK_ASSUME_ROLE_ARN);
    final String dynamodbEndpoint = configuration.getString(AWSInput.CK_DYNAMODB_ENDPOINT);
    final String cloudwatchEndpoint = configuration.getString(AWSInput.CK_CLOUDWATCH_ENDPOINT);
    final String iamEndpoint = configuration.getString(AWSInput.CK_IAM_ENDPOINT);
    final String kinesisEndpoint = configuration.getString(AWSInput.CK_KINESIS_ENDPOINT);
    // Validate the endpoints, because the Kinesis client will silently fail if an endpoint is invalid.
    validateEndpoint(dynamodbEndpoint, "DynamoDB");
    validateEndpoint(cloudwatchEndpoint, "CloudWatch");
    validateEndpoint(iamEndpoint, "IAM");
    validateEndpoint(iamEndpoint, "Kinesis");
    final AWSRequest awsRequest = AWSRequestImpl.builder().region(region.id()).awsAccessKeyId(key).awsSecretAccessKey(secret).assumeRoleArn(assumeRoleArn).cloudwatchEndpoint(cloudwatchEndpoint).dynamodbEndpoint(dynamodbEndpoint).iamEndpoint(iamEndpoint).kinesisEndpoint(kinesisEndpoint).build();
    final int batchSize = configuration.getInt(CK_KINESIS_RECORD_BATCH_SIZE, DEFAULT_BATCH_SIZE);
    final String streamName = configuration.getString(CK_KINESIS_STREAM_NAME);
    final AWSMessageType awsMessageType = AWSMessageType.valueOf(configuration.getString(AWSCodec.CK_AWS_MESSAGE_TYPE));
    this.kinesisConsumer = new KinesisConsumer(nodeId, this, objectMapper, kinesisCallback(input), streamName, awsMessageType, batchSize, awsRequest);
    LOG.debug("Starting Kinesis reader thread for input [{}/{}]", input.getName(), input.getId());
    executor.submit(this.kinesisConsumer);
}

Example 5 with AWSMessageType

use of org.graylog.integrations.aws.AWSMessageType in project graylog-plugin-integrations by Graylog2.

the class AWSService method saveInput.

/**
 * Save the AWS Input
 *
 * This method takes the individual input params in the {@link AWSInputCreateRequest} and creates/saves
 * an input with them.
 */
public Input saveInput(AWSInputCreateRequest request, User user) throws Exception {
    // Transpose the SaveAWSInputRequest to the needed InputCreateRequest
    final HashMap<String, Object> configuration = new HashMap<>();
    configuration.put(AWSCodec.CK_AWS_MESSAGE_TYPE, request.awsMessageType());
    configuration.put(AWSInput.CK_GLOBAL, request.global());
    configuration.put(ThrottleableTransport.CK_THROTTLING_ALLOWED, request.throttlingAllowed());
    configuration.put(AWSCodec.CK_FLOW_LOG_PREFIX, request.addFlowLogPrefix());
    configuration.put(AWSInput.CK_AWS_REGION, request.region());
    configuration.put(AWSInput.CK_ACCESS_KEY, request.awsAccessKeyId());
    configuration.put(AWSInput.CK_SECRET_KEY, request.awsSecretAccessKey());
    configuration.put(AWSInput.CK_ASSUME_ROLE_ARN, request.assumeRoleArn());
    configuration.put(AWSInput.CK_CLOUDWATCH_ENDPOINT, request.cloudwatchEndpoint());
    configuration.put(AWSInput.CK_DYNAMODB_ENDPOINT, request.dynamodbEndpoint());
    configuration.put(AWSInput.CK_IAM_ENDPOINT, request.iamEndpoint());
    configuration.put(AWSInput.CK_KINESIS_ENDPOINT, request.kinesisEndpoint());
    AWSMessageType inputType = AWSMessageType.valueOf(request.awsMessageType());
    if (inputType.isKinesis()) {
        configuration.put(KinesisTransport.CK_KINESIS_STREAM_NAME, request.streamName());
        configuration.put(KinesisTransport.CK_KINESIS_RECORD_BATCH_SIZE, request.batchSize());
    } else {
        throw new Exception("The specified input type is not supported.");
    }
    // Create and save the input.
    final InputCreateRequest inputCreateRequest = InputCreateRequest.create(request.name(), AWSInput.TYPE, false, configuration, nodeId.toString());
    try {
        final MessageInput messageInput = messageInputFactory.create(inputCreateRequest, user.getName(), nodeId.toString());
        messageInput.checkConfiguration();
        final Input input = this.inputService.create(messageInput.asMap());
        final String newInputId = inputService.save(input);
        LOG.debug("New AWS input created. id [{}] request [{}]", newInputId, request);
        return input;
    } catch (NoSuchInputTypeException e) {
        LOG.error("There is no such input type registered.", e);
        throw new NotFoundException("There is no such input type registered.", e);
    } catch (ConfigurationException e) {
        LOG.error("Missing or invalid input configuration.", e);
        throw new BadRequestException("Missing or invalid input configuration.", e);
    }
}