Examples with Codec org.graylog2.plugin.inputs.codecs.Codec used on opensource projects

Example 1 with Codec

use of org.graylog2.plugin.inputs.codecs.Codec in project graylog2-server by Graylog2.

the class RawMessage method encode.

public byte[] encode() {
    try {
        final JournalMessages.CodecInfo codec = msgBuilder.getCodec();
        final JournalMessages.CodecInfo.Builder builder = JournalMessages.CodecInfo.newBuilder(codec);
        final String codecConfigJson = codecConfig.serializeToJson();
        if (codecConfigJson != null) {
            builder.setConfig(codecConfigJson);
        }
        msgBuilder.setCodec(builder.build());
        final JournalMessage journalMessage = msgBuilder.build();
        return journalMessage.toByteArray();
    } catch (UninitializedMessageException e) {
        log.error("Unable to write RawMessage to journal because required fields are missing, " + "this message will be discarded. This is a bug.", e);
        return null;
    }
}

Example 2 with Codec

use of org.graylog2.plugin.inputs.codecs.Codec in project graylog2-server by Graylog2.

the class DecodingProcessor method processMessage.

private void processMessage(final MessageEvent event) throws ExecutionException {
    final RawMessage raw = event.getRaw();
    // for backwards compatibility: the last source node should contain the input we use.
    // this means that extractors etc defined on the prior inputs are silently ignored.
    // TODO fix the above
    String inputIdOnCurrentNode;
    try {
        // .inputId checked during raw message decode!
        inputIdOnCurrentNode = Iterables.getLast(raw.getSourceNodes()).inputId;
    } catch (NoSuchElementException e) {
        inputIdOnCurrentNode = null;
    }
    final Codec.Factory<? extends Codec> factory = codecFactory.get(raw.getCodecName());
    if (factory == null) {
        LOG.warn("Couldn't find factory for codec <{}>, skipping message {} on input <{}>.", raw.getCodecName(), raw, inputIdOnCurrentNode);
        return;
    }
    final Codec codec = factory.create(raw.getCodecConfig());
    final String baseMetricName = name(codec.getClass(), inputIdOnCurrentNode);
    Message message = null;
    Collection<Message> messages = null;
    final Timer.Context decodeTimeCtx = parseTime.time();
    final long decodeTime;
    try {
        // TODO The Codec interface should be changed for 2.0 to support collections of messages so we can remove this hack.
        if (codec instanceof MultiMessageCodec) {
            messages = ((MultiMessageCodec) codec).decodeMessages(raw);
        } else {
            message = codec.decode(raw);
        }
    } catch (RuntimeException e) {
        LOG.error("Unable to decode raw message {} on input <{}>.", raw, inputIdOnCurrentNode);
        metricRegistry.meter(name(baseMetricName, "failures")).mark();
        throw e;
    } finally {
        decodeTime = decodeTimeCtx.stop();
    }
    if (message != null) {
        event.setMessage(postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, message, decodeTime));
    } else if (messages != null && !messages.isEmpty()) {
        final List<Message> processedMessages = Lists.newArrayListWithCapacity(messages.size());
        for (final Message msg : messages) {
            final Message processedMessage = postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, msg, decodeTime);
            if (processedMessage != null) {
                processedMessages.add(processedMessage);
            }
        }
        event.setMessages(processedMessages);
    }
}

Example 3 with Codec

use of org.graylog2.plugin.inputs.codecs.Codec in project graylog2-server by Graylog2.

the class DecodingProcessor method postProcessMessage.

@Nullable
private Message postProcessMessage(RawMessage raw, Codec codec, String inputIdOnCurrentNode, String baseMetricName, Message message, long decodeTime) {
    if (message == null) {
        metricRegistry.meter(name(baseMetricName, "failures")).mark();
        return null;
    }
    if (!message.isComplete()) {
        metricRegistry.meter(name(baseMetricName, "incomplete")).mark();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Dropping incomplete message {} on input <{}>. Parsed fields: [{}]", raw, inputIdOnCurrentNode, message.getFields());
        }
        return null;
    }
    message.setJournalOffset(raw.getJournalOffset());
    message.recordTiming(serverStatus, "parse", decodeTime);
    metricRegistry.timer(name(baseMetricName, "parseTime")).update(decodeTime, TimeUnit.NANOSECONDS);
    for (final RawMessage.SourceNode node : raw.getSourceNodes()) {
        switch(node.type) {
            case SERVER:
                // Always use the last source node.
                if (message.getField("gl2_source_input") != null) {
                    LOG.debug("Multiple server nodes ({} {}) set for message id {}", message.getField("gl2_source_input"), node.nodeId, message.getId());
                }
                message.addField("gl2_source_input", node.inputId);
                message.addField("gl2_source_node", node.nodeId);
                break;
            // TODO Due to be removed in Graylog 3.x
            case RADIO:
                // Always use the last source node.
                if (message.getField("gl2_source_radio_input") != null) {
                    LOG.debug("Multiple radio nodes ({} {}) set for message id {}", message.getField("gl2_source_radio_input"), node.nodeId, message.getId());
                }
                message.addField("gl2_source_radio_input", node.inputId);
                message.addField("gl2_source_radio", node.nodeId);
                break;
        }
    }
    if (inputIdOnCurrentNode != null) {
        try {
            message.setSourceInputId(inputIdOnCurrentNode);
        } catch (RuntimeException e) {
            LOG.warn("Unable to find input with id " + inputIdOnCurrentNode + ", not setting input id in this message.", e);
        }
    }
    final ResolvableInetSocketAddress remoteAddress = raw.getRemoteAddress();
    if (remoteAddress != null) {
        final String addrString = InetAddresses.toAddrString(remoteAddress.getAddress());
        message.addField("gl2_remote_ip", addrString);
        if (remoteAddress.getPort() > 0) {
            message.addField("gl2_remote_port", remoteAddress.getPort());
        }
        if (remoteAddress.isReverseLookedUp()) {
            // avoid reverse lookup if the hostname is available
            message.addField("gl2_remote_hostname", remoteAddress.getHostName());
        }
        if (Strings.isNullOrEmpty(message.getSource())) {
            message.setSource(addrString);
        }
    }
    if (codec.getConfiguration() != null && codec.getConfiguration().stringIsSet(Codec.Config.CK_OVERRIDE_SOURCE)) {
        message.setSource(codec.getConfiguration().getString(Codec.Config.CK_OVERRIDE_SOURCE));
    }
    // Make sure that there is a value for the source field.
    if (Strings.isNullOrEmpty(message.getSource())) {
        message.setSource("unknown");
    }
    metricRegistry.meter(name(baseMetricName, "processedMessages")).mark();
    return message;
}

Example 4 with Codec

use of org.graylog2.plugin.inputs.codecs.Codec in project graylog2-server by Graylog2.

the class MessageResource method parse.

@POST
@Path("/parse")
@Timed
@Consumes(MediaType.APPLICATION_JSON)
@ApiOperation(value = "Parse a raw message")
@ApiResponses(value = { @ApiResponse(code = 404, message = "Specified codec does not exist."), @ApiResponse(code = 400, message = "Could not decode message.") })
@NoAuditEvent("only used to parse a test message")
public ResultMessage parse(@ApiParam(name = "JSON body", required = true) MessageParseRequest request) {
    Codec codec;
    try {
        final Configuration configuration = new Configuration(request.configuration());
        codec = codecFactory.create(request.codec(), configuration);
    } catch (IllegalArgumentException e) {
        throw new NotFoundException(e);
    }
    final ResolvableInetSocketAddress remoteAddress = ResolvableInetSocketAddress.wrap(new InetSocketAddress(request.remoteAddress(), 1234));
    final RawMessage rawMessage = new RawMessage(0, new UUID(), Tools.nowUTC(), remoteAddress, request.message().getBytes(StandardCharsets.UTF_8));
    final Message message = decodeMessage(codec, remoteAddress, rawMessage);
    return ResultMessage.createFromMessage(message);
}

Example 5 with Codec

use of org.graylog2.plugin.inputs.codecs.Codec in project graylog2-server by Graylog2.

the class MessageResource method decodeMessage.

private Message decodeMessage(Codec codec, ResolvableInetSocketAddress remoteAddress, RawMessage rawMessage) {
    Message message;
    try {
        message = codec.decode(rawMessage);
    } catch (Exception e) {
        throw new BadRequestException("Could not decode message");
    }
    if (message == null) {
        throw new BadRequestException("Could not decode message");
    }
    // Ensure the decoded Message has a source, otherwise creating a ResultMessage will fail
    if (isNullOrEmpty(message.getSource())) {
        final String address = InetAddresses.toAddrString(remoteAddress.getAddress());
        message.setSource(address);
    }
    // Override source
    final Configuration configuration = codec.getConfiguration();
    if (configuration.stringIsSet(Codec.Config.CK_OVERRIDE_SOURCE)) {
        message.setSource(configuration.getString(Codec.Config.CK_OVERRIDE_SOURCE));
    }
    return message;
}