Examples with Messages org.graylog2.plugin.Messages used on opensource projects

Example 31 with Messages

use of org.graylog2.plugin.Messages in project graylog2-server by Graylog2.

the class V20161116172200_CreateDefaultStreamMigration method createDefaultStream.

private void createDefaultStream() {
    final IndexSet indexSet = indexSetRegistry.getDefault();
    final ObjectId id = new ObjectId(Stream.DEFAULT_STREAM_ID);
    final Map<String, Object> fields = ImmutableMap.<String, Object>builder().put(StreamImpl.FIELD_TITLE, "All messages").put(StreamImpl.FIELD_DESCRIPTION, "Stream containing all messages").put(StreamImpl.FIELD_DISABLED, false).put(StreamImpl.FIELD_CREATED_AT, DateTime.now(DateTimeZone.UTC)).put(StreamImpl.FIELD_CREATOR_USER_ID, "local:admin").put(StreamImpl.FIELD_MATCHING_TYPE, StreamImpl.MatchingType.DEFAULT.name()).put(StreamImpl.FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM, false).put(StreamImpl.FIELD_DEFAULT_STREAM, true).put(StreamImpl.FIELD_INDEX_SET_ID, indexSet.getConfig().id()).build();
    final Stream stream = new StreamImpl(id, fields, Collections.emptyList(), Collections.emptySet(), indexSet);
    try {
        streamService.save(stream);
        LOG.info("Successfully created default stream: {}", stream.getTitle());
    } catch (ValidationException e) {
        LOG.error("Couldn't create default stream! This is a bug!");
    }
    clusterEventBus.post(StreamsChangedEvent.create(stream.getId()));
}

Example 32 with Messages

use of org.graylog2.plugin.Messages in project graylog2-server by Graylog2.

the class DiscardMessageOutput method write.

@Override
public void write(List<Message> messages) throws Exception {
    long maxOffset = Long.MIN_VALUE;
    for (final Message message : messages) {
        maxOffset = Math.max(message.getJournalOffset(), maxOffset);
    }
    journal.markJournalOffsetCommitted(maxOffset);
    messagesDiscarded.mark(messages.size());
}

Example 33 with Messages

use of org.graylog2.plugin.Messages in project graylog2-server by Graylog2.

the class JournalReader method run.

@Override
protected void run() throws Exception {
    try {
        requestedReadCount = metricRegistry.register(name(this.getClass(), "requestedReadCount"), new HdrHistogram(processBuffer.getRingBufferSize() + 1, 3));
    } catch (IllegalArgumentException e) {
        log.warn("Metric already exists", e);
        throw e;
    }
    while (isRunning()) {
        // TODO interfere with reading if we are not 100% certain we should be reading, see #listenForLifecycleChanges
        if (!shouldBeReading) {
            Uninterruptibles.sleepUninterruptibly(100, MILLISECONDS);
            // don't read immediately, but check if we should be shutting down.
            continue;
        }
        // approximate count to read from the journal to backfill the processing chain
        final long remainingCapacity = processBuffer.getRemainingCapacity();
        requestedReadCount.update(remainingCapacity);
        final List<Journal.JournalReadEntry> encodedRawMessages = journal.read(remainingCapacity);
        if (encodedRawMessages.isEmpty()) {
            log.debug("No messages to read from Journal, waiting until the writer adds more messages.");
            // block until something is written to the journal again
            try {
                readBlocked.inc();
                journalFilled.acquire();
            } catch (InterruptedException ignored) {
                // this can happen when we are blocked but the system wants to shut down. We don't have to do anything in that case.
                continue;
            }
            log.debug("Messages have been written to Journal, continuing to read.");
            // we don't care how many messages were inserted in the meantime, we'll read all of them eventually
            journalFilled.drainPermits();
        } else {
            readMessages.mark(encodedRawMessages.size());
            log.debug("Processing {} messages from journal.", encodedRawMessages.size());
            for (final Journal.JournalReadEntry encodedRawMessage : encodedRawMessages) {
                final RawMessage rawMessage = RawMessage.decode(encodedRawMessage.getPayload(), encodedRawMessage.getOffset());
                if (rawMessage == null) {
                    // never insert null objects into the ringbuffer, as that is useless
                    log.error("Found null raw message!");
                    journal.markJournalOffsetCommitted(encodedRawMessage.getOffset());
                    continue;
                }
                processBuffer.insertBlocking(rawMessage);
            }
        }
    }
    log.info("Stopping.");
}

Example 34 with Messages

use of org.graylog2.plugin.Messages in project graylog2-server by Graylog2.

the class KafkaJournal method write.

/**
     * Writes the list of entries to the journal.
     *
     * @param entries journal entries to be written
     * @return the last position written to in the journal
     */
@Override
public long write(List<Entry> entries) {
    try (Timer.Context ignored = writeTime.time()) {
        long payloadSize = 0L;
        long messageSetSize = 0L;
        long lastWriteOffset = 0L;
        final List<Message> messages = new ArrayList<>(entries.size());
        for (final Entry entry : entries) {
            final byte[] messageBytes = entry.getMessageBytes();
            final byte[] idBytes = entry.getIdBytes();
            payloadSize += messageBytes.length;
            final Message newMessage = new Message(messageBytes, idBytes);
            // Calculate the size of the new message in the message set by including the overhead for the log entry.
            final int newMessageSize = MessageSet.entrySize(newMessage);
            if (newMessageSize > maxMessageSize) {
                writeDiscardedMessages.mark();
                LOG.warn("Message with ID <{}> is too large to store in journal, skipping! (size: {} bytes / max: {} bytes)", new String(idBytes, StandardCharsets.UTF_8), newMessageSize, maxMessageSize);
                payloadSize = 0;
                continue;
            }
            // list of message to avoid a MessageSetSizeTooLargeException.
            if ((messageSetSize + newMessageSize) > maxSegmentSize) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Flushing {} bytes message set with {} messages to avoid overflowing segment with max size of {} bytes", messageSetSize, messages.size(), maxSegmentSize);
                }
                lastWriteOffset = flushMessages(messages, payloadSize);
                // Reset the messages list and size counters to start a new batch.
                messages.clear();
                messageSetSize = 0;
                payloadSize = 0;
            }
            messages.add(newMessage);
            messageSetSize += newMessageSize;
            if (LOG.isTraceEnabled()) {
                LOG.trace("Message {} contains bytes {}", bytesToHex(idBytes), bytesToHex(messageBytes));
            }
        }
        // Flush the rest of the messages.
        if (messages.size() > 0) {
            lastWriteOffset = flushMessages(messages, payloadSize);
        }
        return lastWriteOffset;
    }
}

Example 35 with Messages

use of org.graylog2.plugin.Messages in project graylog2-server by Graylog2.

the class KafkaJournal method read.

public List<JournalReadEntry> read(long readOffset, long requestedMaximumCount) {
    // Always read at least one!
    final long maximumCount = Math.max(1, requestedMaximumCount);
    long maxOffset = readOffset + maximumCount;
    if (shuttingDown) {
        return Collections.emptyList();
    }
    final List<JournalReadEntry> messages = new ArrayList<>(Ints.saturatedCast(maximumCount));
    try (Timer.Context ignored = readTime.time()) {
        final long logStartOffset = getLogStartOffset();
        if (readOffset < logStartOffset) {
            LOG.info("Read offset {} before start of log at {}, starting to read from the beginning of the journal.", readOffset, logStartOffset);
            readOffset = logStartOffset;
            maxOffset = readOffset + maximumCount;
        }
        LOG.debug("Requesting to read a maximum of {} messages (or 5MB) from the journal, offset interval [{}, {})", maximumCount, readOffset, maxOffset);
        // TODO benchmark and make read-ahead strategy configurable for performance tuning
        final MessageSet messageSet = kafkaLog.read(readOffset, 5 * 1024 * 1024, Option.<Object>apply(maxOffset)).messageSet();
        final Iterator<MessageAndOffset> iterator = messageSet.iterator();
        long firstOffset = Long.MIN_VALUE;
        long lastOffset = Long.MIN_VALUE;
        long totalBytes = 0;
        while (iterator.hasNext()) {
            final MessageAndOffset messageAndOffset = iterator.next();
            if (firstOffset == Long.MIN_VALUE)
                firstOffset = messageAndOffset.offset();
            // always remember the last seen offset for debug purposes below
            lastOffset = messageAndOffset.offset();
            final byte[] payloadBytes = ByteBufferUtils.readBytes(messageAndOffset.message().payload());
            if (LOG.isTraceEnabled()) {
                final byte[] keyBytes = ByteBufferUtils.readBytes(messageAndOffset.message().key());
                LOG.trace("Read message {} contains {}", bytesToHex(keyBytes), bytesToHex(payloadBytes));
            }
            totalBytes += payloadBytes.length;
            messages.add(new JournalReadEntry(payloadBytes, messageAndOffset.offset()));
            // remember where to read from
            nextReadOffset = messageAndOffset.nextOffset();
        }
        if (messages.isEmpty()) {
            LOG.debug("No messages available to read for offset interval [{}, {}).", readOffset, maxOffset);
        } else {
            LOG.debug("Read {} messages, total payload size {}, from journal, offset interval [{}, {}], requested read at {}", messages.size(), totalBytes, firstOffset, lastOffset, readOffset);
        }
    } catch (OffsetOutOfRangeException e) {
        // This is fine, the reader tries to read faster than the writer committed data. Next read will get the data.
        LOG.debug("Offset out of range, no messages available starting at offset {}", readOffset);
    } catch (Exception e) {
        // sigh.
        if (shuttingDown) {
            LOG.debug("Caught exception during shutdown, ignoring it because we might have been blocked on a read.");
            return Collections.emptyList();
        }
        //noinspection ConstantConditions
        if (e instanceof ClosedByInterruptException) {
            LOG.debug("Interrupted while reading from journal, during shutdown this is harmless and ignored.", e);
        } else {
            throw e;
        }
    }
    readMessages.mark(messages.size());
    return messages;
}