Examples with Messages org.graylog2.indexer.messages.Messages used on opensource projects

Example 6 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class LocalKafkaJournal method write.

/**
 * Writes the list of entries to the journal.
 *
 * @param entries journal entries to be written
 * @return the last position written to in the journal
 */
@Override
public long write(List<Entry> entries) {
    try (Timer.Context ignored = writeTime.time()) {
        long payloadSize = 0L;
        long messageSetSize = 0L;
        long lastWriteOffset = 0L;
        final List<Message> messages = new ArrayList<>(entries.size());
        for (final Entry entry : entries) {
            final byte[] messageBytes = entry.getMessageBytes();
            final byte[] idBytes = entry.getIdBytes();
            payloadSize += messageBytes.length;
            final Message newMessage = new Message(messageBytes, idBytes);
            // Calculate the size of the new message in the message set by including the overhead for the log entry.
            final int newMessageSize = MessageSet.entrySize(newMessage);
            if (newMessageSize > maxMessageSize) {
                writeDiscardedMessages.mark();
                LOG.warn("Message with ID <{}> is too large to store in journal, skipping! (size: {} bytes / max: {} bytes)", new String(idBytes, StandardCharsets.UTF_8), newMessageSize, maxMessageSize);
                payloadSize = 0;
                continue;
            }
            // list of message to avoid a MessageSetSizeTooLargeException.
            if ((messageSetSize + newMessageSize) > maxSegmentSize) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Flushing {} bytes message set with {} messages to avoid overflowing segment with max size of {} bytes", messageSetSize, messages.size(), maxSegmentSize);
                }
                lastWriteOffset = flushMessages(messages, payloadSize);
                // Reset the messages list and size counters to start a new batch.
                messages.clear();
                messageSetSize = 0;
                payloadSize = 0;
            }
            messages.add(newMessage);
            messageSetSize += newMessageSize;
            if (LOG.isTraceEnabled()) {
                LOG.trace("Message {} contains bytes {}", bytesToHex(idBytes), bytesToHex(messageBytes));
            }
        }
        // Flush the rest of the messages.
        if (messages.size() > 0) {
            lastWriteOffset = flushMessages(messages, payloadSize);
        }
        return lastWriteOffset;
    }
}

Example 7 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class LocalKafkaJournal method read.

/**
 * Read from the journal, starting at the given offset. If the underlying journal implementation returns an empty
 * list of entries, it will be returned even if we know there are more entries in the journal.
 *
 * @param readOffset Offset to start reading at
 * @param requestedMaximumCount Maximum number of entries to return.
 * @return A list of entries
 */
public List<JournalReadEntry> read(long readOffset, long requestedMaximumCount) {
    // Always read at least one!
    final long maximumCount = Math.max(1, requestedMaximumCount);
    long maxOffset = readOffset + maximumCount;
    if (shuttingDown) {
        return Collections.emptyList();
    }
    final List<JournalReadEntry> messages = new ArrayList<>(Ints.saturatedCast(maximumCount));
    try (Timer.Context ignored = readTime.time()) {
        final long logStartOffset = getLogStartOffset();
        if (readOffset < logStartOffset) {
            LOG.info("Read offset {} before start of log at {}, starting to read from the beginning of the journal.", readOffset, logStartOffset);
            readOffset = logStartOffset;
            maxOffset = readOffset + maximumCount;
        }
        LOG.debug("Requesting to read a maximum of {} messages (or 5MB) from the journal, offset interval [{}, {})", maximumCount, readOffset, maxOffset);
        // TODO benchmark and make read-ahead strategy configurable for performance tuning
        final MessageSet messageSet = kafkaLog.read(readOffset, 5 * 1024 * 1024, Option.<Object>apply(maxOffset)).messageSet();
        final Iterator<MessageAndOffset> iterator = messageSet.iterator();
        long firstOffset = Long.MIN_VALUE;
        long lastOffset = Long.MIN_VALUE;
        long totalBytes = 0;
        while (iterator.hasNext()) {
            final MessageAndOffset messageAndOffset = iterator.next();
            if (firstOffset == Long.MIN_VALUE) {
                firstOffset = messageAndOffset.offset();
            }
            // always remember the last seen offset for debug purposes below
            lastOffset = messageAndOffset.offset();
            final byte[] payloadBytes = ByteBufferUtils.readBytes(messageAndOffset.message().payload());
            if (LOG.isTraceEnabled()) {
                final byte[] keyBytes = ByteBufferUtils.readBytes(messageAndOffset.message().key());
                LOG.trace("Read message {} contains {}", bytesToHex(keyBytes), bytesToHex(payloadBytes));
            }
            totalBytes += payloadBytes.length;
            messages.add(new JournalReadEntry(payloadBytes, messageAndOffset.offset()));
            // remember where to read from
            nextReadOffset = messageAndOffset.nextOffset();
        }
        if (messages.isEmpty()) {
            LOG.debug("No messages available to read for offset interval [{}, {}).", readOffset, maxOffset);
        } else {
            LOG.debug("Read {} messages, total payload size {}, from journal, offset interval [{}, {}], requested read at {}", messages.size(), totalBytes, firstOffset, lastOffset, readOffset);
        }
    } catch (OffsetOutOfRangeException e) {
        // This is fine, the reader tries to read faster than the writer committed data. Next read will get the data.
        LOG.debug("Offset out of range, no messages available starting at offset {}", readOffset);
    } catch (Exception e) {
        // sigh.
        if (shuttingDown) {
            LOG.debug("Caught exception during shutdown, ignoring it because we might have been blocked on a read.");
            return Collections.emptyList();
        }
        // noinspection ConstantConditions
        if (e instanceof ClosedByInterruptException) {
            LOG.debug("Interrupted while reading from journal, during shutdown this is harmless and ignored.", e);
        } else {
            throw e;
        }
    }
    readMessages.mark(messages.size());
    return messages;
}

Example 8 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class DecodingProcessor method processMessage.

private void processMessage(final MessageEvent event) throws ExecutionException {
    final RawMessage raw = event.getRaw();
    // for backwards compatibility: the last source node should contain the input we use.
    // this means that extractors etc defined on the prior inputs are silently ignored.
    // TODO fix the above
    String inputIdOnCurrentNode;
    try {
        // .inputId checked during raw message decode!
        inputIdOnCurrentNode = Iterables.getLast(raw.getSourceNodes()).inputId;
    } catch (NoSuchElementException e) {
        inputIdOnCurrentNode = null;
    }
    final Codec.Factory<? extends Codec> factory = codecFactory.get(raw.getCodecName());
    if (factory == null) {
        LOG.warn("Couldn't find factory for codec <{}>, skipping message {} on input <{}>.", raw.getCodecName(), raw, inputIdOnCurrentNode);
        return;
    }
    final Codec codec = factory.create(raw.getCodecConfig());
    final String baseMetricName = name(codec.getClass(), inputIdOnCurrentNode);
    Message message = null;
    Collection<Message> messages = null;
    final Timer.Context decodeTimeCtx = parseTime.time();
    final long decodeTime;
    try {
        // TODO The Codec interface should be changed for 2.0 to support collections of messages so we can remove this hack.
        if (codec instanceof MultiMessageCodec) {
            messages = ((MultiMessageCodec) codec).decodeMessages(raw);
        } else {
            message = codec.decode(raw);
        }
    } catch (RuntimeException e) {
        LOG.error("Unable to decode raw message {} on input <{}>.", raw, inputIdOnCurrentNode);
        metricRegistry.meter(name(baseMetricName, "failures")).mark();
        throw e;
    } finally {
        decodeTime = decodeTimeCtx.stop();
    }
    if (message != null) {
        event.setMessage(postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, message, decodeTime));
    } else if (messages != null && !messages.isEmpty()) {
        final List<Message> processedMessages = Lists.newArrayListWithCapacity(messages.size());
        for (final Message msg : messages) {
            final Message processedMessage = postProcessMessage(raw, codec, inputIdOnCurrentNode, baseMetricName, msg, decodeTime);
            if (processedMessage != null) {
                processedMessages.add(processedMessage);
            }
        }
        event.setMessages(processedMessages);
    }
}

Example 9 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class ProcessBufferProcessor method handleMessage.

private void handleMessage(@Nonnull Message msg) {
    msg.addStream(defaultStreamProvider.get());
    Messages messages = msg;
    for (MessageProcessor messageProcessor : orderedMessageProcessors) {
        messages = messageProcessor.process(messages);
    }
    for (Message message : messages) {
        message.ensureValidTimestamp();
        if (!message.hasField(Message.FIELD_GL2_MESSAGE_ID) || isNullOrEmpty(message.getFieldAs(String.class, Message.FIELD_GL2_MESSAGE_ID))) {
            // Set the message ID once all message processors have finished
            // See documentation of Message.FIELD_GL2_MESSAGE_ID for details
            message.addField(Message.FIELD_GL2_MESSAGE_ID, ulid.nextULID());
        }
        // The processing time should only be set once all message processors have finished
        message.setProcessingTime(Tools.nowUTC());
        processingStatusRecorder.updatePostProcessingReceiveTime(message.getReceiveTime());
        if (failureSubmissionService.submitProcessingErrors(message)) {
            outputBuffer.insertBlocking(message);
        }
    }
}

Example 10 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class PipelineInterpreterTest method testMatchPassContinuesIfNoRuleMatched.

@Test
public void testMatchPassContinuesIfNoRuleMatched() {
    final RuleService ruleService = mock(MongoDbRuleService.class);
    when(ruleService.loadAll()).thenReturn(ImmutableList.of(RULE_TRUE, RULE_FALSE, RULE_ADD_FOOBAR));
    final PipelineService pipelineService = mock(MongoDbPipelineService.class);
    when(pipelineService.loadAll()).thenReturn(Collections.singleton(PipelineDao.create("p1", "title", "description", "pipeline \"pipeline\"\n" + "stage 0 match pass\n" + "    rule \"false\";\n" + "stage 1 match pass\n" + "    rule \"add_foobar\";\n" + "end\n", Tools.nowUTC(), null)));
    final Map<String, Function<?>> functions = ImmutableMap.of(SetField.NAME, new SetField());
    final PipelineInterpreter interpreter = createPipelineInterpreter(ruleService, pipelineService, functions);
    final Messages processed = interpreter.process(messageInDefaultStream("message", "test"));
    final List<Message> messages = ImmutableList.copyOf(processed);
    assertThat(messages).hasSize(1);
    final Message actualMessage = messages.get(0);
    assertThat(actualMessage.getFieldAs(String.class, "foobar")).isEqualTo("covfefe");
}