Examples with MessageEvent org.graylog2.plugin.buffers.MessageEvent used on opensource projects

Example 1 with MessageEvent

use of org.graylog2.plugin.buffers.MessageEvent in project graylog2-server by Graylog2.

the class DecodingProcessor method processMessage.

private void processMessage(final MessageEvent event) throws ExecutionException {
    final RawMessage raw = event.getRaw();
    // for backwards compatibility: the last source node should contain the input we use.
    // this means that extractors etc defined on the prior inputs are silently ignored.
    // TODO fix the above
    String inputIdOnCurrentNode;
    try {
        // .inputId checked during raw message decode!
        inputIdOnCurrentNode = Iterables.getLast(raw.getSourceNodes()).inputId;
    } catch (NoSuchElementException e) {
        inputIdOnCurrentNode = null;
    }
    final Codec.Factory<? extends Codec> factory = codecFactory.get(raw.getCodecName());
    if (factory == null) {
        LOG.warn("Couldn't find factory for codec <{}>, skipping message {} on input <{}>.", raw.getCodecName(), raw, inputIdOnCurrentNode);
        return;
    }
    final Codec codec = factory.create(raw.getCodecConfig());
    final String baseMetricName = name(codec.getClass(), inputIdOnCurrentNode);
    Message message = null;
    Collection<Message> messages = null;
    final Timer.Context decodeTimeCtx = parseTime.time();
    final long decodeTime;
    try {
        // TODO The Codec interface should be changed for 2.0 to support collections of messages so we can remove this hack.
        if (codec instanceof MultiMessageCodec) {
            messages = ((MultiMessageCodec) codec).decodeMessages(raw);
        } else {
            message = codec.decode(raw);
        }
    } catch (RuntimeException e) {
        LOG.error("Unable to decode raw message {} on input <{}>.", raw, inputIdOnCurrentNode);
        metricRegistry.meter(name(baseMetricName, "failures")).mark();
        throw e;
    } finally {
        decodeTime = decodeTimeCtx.stop();
    }
    if (message != null) {
        event.setMessage(postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, message, decodeTime));
    } else if (messages != null && !messages.isEmpty()) {
        final List<Message> processedMessages = Lists.newArrayListWithCapacity(messages.size());
        for (final Message msg : messages) {
            final Message processedMessage = postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, msg, decodeTime);
            if (processedMessage != null) {
                processedMessages.add(processedMessage);
            }
        }
        event.setMessages(processedMessages);
    }
}

Example 2 with MessageEvent

use of org.graylog2.plugin.buffers.MessageEvent in project graylog2-server by Graylog2.

the class OutputBufferProcessor method onEvent.

/**
 * Each message will be written to one or more outputs.
 * <p>
 * The default output is always being used for every message, but optionally the message can be routed to additional
 * outputs, currently based on the stream outputs that are configured in the system.
 * </p>
 * <p>
 * The stream outputs are time limited so one bad output does not impact throughput too much. Essentially this means
 * that the work of writing to the outputs is performed, but the writer threads will not wait forever for stream
 * outputs to finish their work. <b>This might lead to increased memory usage!</b>
 * </p>
 * <p>
 * The default output, however, is allowed to block and is not subject to time limiting. This is important because it
 * can exert back pressure on the processing pipeline this way, making sure we don't run into excessive heap usage.
 * </p>
 *
 * @param event the message to write to outputs
 * @throws Exception
 */
@Override
public void onEvent(MessageEvent event) throws Exception {
    incomingMessages.mark();
    final Message msg = event.getMessage();
    if (msg == null) {
        LOG.debug("Skipping null message.");
        return;
    }
    LOG.debug("Processing message <{}> from OutputBuffer.", msg.getId());
    final Set<MessageOutput> messageOutputs = outputRouter.getStreamOutputsForMessage(msg);
    msg.recordCounter(serverStatus, "matched-outputs", messageOutputs.size());
    final Future<?> defaultOutputCompletion = processMessage(msg, defaultMessageOutput);
    final CountDownLatch streamOutputsDoneSignal = new CountDownLatch(messageOutputs.size());
    for (final MessageOutput output : messageOutputs) {
        processMessage(msg, output, streamOutputsDoneSignal);
    }
    // Wait until all writer threads for stream outputs have finished or timeout is reached.
    if (!streamOutputsDoneSignal.await(configuration.getOutputModuleTimeout(), TimeUnit.MILLISECONDS)) {
        LOG.warn("Timeout reached. Not waiting any longer for stream output writer threads to complete.");
    }
    // this exerts the back pressure to the system
    if (defaultOutputCompletion != null) {
        Uninterruptibles.getUninterruptibly(defaultOutputCompletion);
    } else {
        LOG.error("The default output future was null, this is a bug!");
    }
    if (msg.hasRecordings()) {
        LOG.debug("Message event trace: {}", msg.recordingsAsString());
    }
    outputThroughput.inc();
    LOG.debug("Wrote message <{}> to all outputs. Finished handling.", msg.getId());
    event.clearMessages();
}

Example 3 with MessageEvent

use of org.graylog2.plugin.buffers.MessageEvent in project graylog2-server by Graylog2.

the class ProcessBuffer method insertBlocking.

public void insertBlocking(@Nonnull RawMessage rawMessage) {
    final long sequence = ringBuffer.next();
    final MessageEvent event = ringBuffer.get(sequence);
    event.setRaw(rawMessage);
    ringBuffer.publish(sequence);
    afterInsert(1);
}