Examples with Messages org.graylog2.indexer.messages.Messages used on opensource projects

Example 46 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class AbsoluteSearchResource method searchAbsolute.

@GET
@Timed
@ApiOperation(value = "Message search with absolute timerange.", notes = "Search for messages using an absolute timerange, specified as from/to " + "with format yyyy-MM-ddTHH:mm:ss.SSSZ (e.g. 2014-01-23T15:34:49.000Z) or yyyy-MM-dd HH:mm:ss.")
@Produces(MediaType.APPLICATION_JSON)
@ApiResponses(value = { @ApiResponse(code = 400, message = "Invalid timerange parameters provided.") })
public SearchResponse searchAbsolute(@ApiParam(name = "query", value = "Query (Lucene syntax)", required = true) @QueryParam("query") @NotEmpty String query, @ApiParam(name = "from", value = "Timerange start. See description for date format", required = true) @QueryParam("from") String from, @ApiParam(name = "to", value = "Timerange end. See description for date format", required = true) @QueryParam("to") String to, @ApiParam(name = "limit", value = "Maximum number of messages to return.", required = false) @QueryParam("limit") int limit, @ApiParam(name = "offset", value = "Offset", required = false) @QueryParam("offset") int offset, @ApiParam(name = "filter", value = "Filter", required = false) @QueryParam("filter") String filter, @ApiParam(name = "fields", value = "Comma separated list of fields to return", required = false) @QueryParam("fields") String fields, @ApiParam(name = "sort", value = "Sorting (field:asc / field:desc)", required = false) @QueryParam("sort") String sort, @ApiParam(name = "decorate", value = "Run decorators on search result", required = false) @QueryParam("decorate") @DefaultValue("true") boolean decorate) {
    checkSearchPermission(filter, RestPermissions.SEARCHES_ABSOLUTE);
    final Sorting sorting = buildSorting(sort);
    final List<String> fieldList = parseOptionalFields(fields);
    TimeRange timeRange = buildAbsoluteTimeRange(from, to);
    final SearchesConfig searchesConfig = SearchesConfig.builder().query(query).filter(filter).fields(fieldList).range(timeRange).limit(limit).offset(offset).sorting(sorting).build();
    final Optional<String> streamId = Searches.extractStreamId(filter);
    try {
        return buildSearchResponse(searches.search(searchesConfig), timeRange, decorate, streamId);
    } catch (SearchPhaseExecutionException e) {
        throw createRequestExceptionForParseFailure(query, e);
    }
}

Example 47 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class DiscardMessageOutput method write.

@Override
public void write(List<Message> messages) throws Exception {
    long maxOffset = Long.MIN_VALUE;
    for (final Message message : messages) {
        maxOffset = Math.max(message.getJournalOffset(), maxOffset);
    }
    journal.markJournalOffsetCommitted(maxOffset);
    messagesDiscarded.mark(messages.size());
}

Example 48 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class JournalReader method run.

@Override
protected void run() throws Exception {
    try {
        requestedReadCount = metricRegistry.register(name(this.getClass(), "requestedReadCount"), new HdrHistogram(processBuffer.getRingBufferSize() + 1, 3));
    } catch (IllegalArgumentException e) {
        log.warn("Metric already exists", e);
        throw e;
    }
    while (isRunning()) {
        // TODO interfere with reading if we are not 100% certain we should be reading, see #listenForLifecycleChanges
        if (!shouldBeReading) {
            Uninterruptibles.sleepUninterruptibly(100, MILLISECONDS);
            // don't read immediately, but check if we should be shutting down.
            continue;
        }
        // approximate count to read from the journal to backfill the processing chain
        final long remainingCapacity = processBuffer.getRemainingCapacity();
        requestedReadCount.update(remainingCapacity);
        final List<Journal.JournalReadEntry> encodedRawMessages = journal.read(remainingCapacity);
        if (encodedRawMessages.isEmpty()) {
            log.debug("No messages to read from Journal, waiting until the writer adds more messages.");
            // block until something is written to the journal again
            try {
                readBlocked.inc();
                journalFilled.acquire();
            } catch (InterruptedException ignored) {
                // this can happen when we are blocked but the system wants to shut down. We don't have to do anything in that case.
                continue;
            }
            log.debug("Messages have been written to Journal, continuing to read.");
            // we don't care how many messages were inserted in the meantime, we'll read all of them eventually
            journalFilled.drainPermits();
        } else {
            readMessages.mark(encodedRawMessages.size());
            log.debug("Processing {} messages from journal.", encodedRawMessages.size());
            for (final Journal.JournalReadEntry encodedRawMessage : encodedRawMessages) {
                final RawMessage rawMessage = RawMessage.decode(encodedRawMessage.getPayload(), encodedRawMessage.getOffset());
                if (rawMessage == null) {
                    // never insert null objects into the ringbuffer, as that is useless
                    log.error("Found null raw message!");
                    journal.markJournalOffsetCommitted(encodedRawMessage.getOffset());
                    continue;
                }
                processBuffer.insertBlocking(rawMessage);
            }
        }
    }
    log.info("Stopping.");
}

Example 49 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class KafkaJournal method write.

/**
     * Writes the list of entries to the journal.
     *
     * @param entries journal entries to be written
     * @return the last position written to in the journal
     */
@Override
public long write(List<Entry> entries) {
    try (Timer.Context ignored = writeTime.time()) {
        long payloadSize = 0L;
        long messageSetSize = 0L;
        long lastWriteOffset = 0L;
        final List<Message> messages = new ArrayList<>(entries.size());
        for (final Entry entry : entries) {
            final byte[] messageBytes = entry.getMessageBytes();
            final byte[] idBytes = entry.getIdBytes();
            payloadSize += messageBytes.length;
            final Message newMessage = new Message(messageBytes, idBytes);
            // Calculate the size of the new message in the message set by including the overhead for the log entry.
            final int newMessageSize = MessageSet.entrySize(newMessage);
            if (newMessageSize > maxMessageSize) {
                writeDiscardedMessages.mark();
                LOG.warn("Message with ID <{}> is too large to store in journal, skipping! (size: {} bytes / max: {} bytes)", new String(idBytes, StandardCharsets.UTF_8), newMessageSize, maxMessageSize);
                payloadSize = 0;
                continue;
            }
            // list of message to avoid a MessageSetSizeTooLargeException.
            if ((messageSetSize + newMessageSize) > maxSegmentSize) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Flushing {} bytes message set with {} messages to avoid overflowing segment with max size of {} bytes", messageSetSize, messages.size(), maxSegmentSize);
                }
                lastWriteOffset = flushMessages(messages, payloadSize);
                // Reset the messages list and size counters to start a new batch.
                messages.clear();
                messageSetSize = 0;
                payloadSize = 0;
            }
            messages.add(newMessage);
            messageSetSize += newMessageSize;
            if (LOG.isTraceEnabled()) {
                LOG.trace("Message {} contains bytes {}", bytesToHex(idBytes), bytesToHex(messageBytes));
            }
        }
        // Flush the rest of the messages.
        if (messages.size() > 0) {
            lastWriteOffset = flushMessages(messages, payloadSize);
        }
        return lastWriteOffset;
    }
}

Example 50 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class KafkaJournal method read.

public List<JournalReadEntry> read(long readOffset, long requestedMaximumCount) {
    // Always read at least one!
    final long maximumCount = Math.max(1, requestedMaximumCount);
    long maxOffset = readOffset + maximumCount;
    if (shuttingDown) {
        return Collections.emptyList();
    }
    final List<JournalReadEntry> messages = new ArrayList<>(Ints.saturatedCast(maximumCount));
    try (Timer.Context ignored = readTime.time()) {
        final long logStartOffset = getLogStartOffset();
        if (readOffset < logStartOffset) {
            LOG.info("Read offset {} before start of log at {}, starting to read from the beginning of the journal.", readOffset, logStartOffset);
            readOffset = logStartOffset;
            maxOffset = readOffset + maximumCount;
        }
        LOG.debug("Requesting to read a maximum of {} messages (or 5MB) from the journal, offset interval [{}, {})", maximumCount, readOffset, maxOffset);
        // TODO benchmark and make read-ahead strategy configurable for performance tuning
        final MessageSet messageSet = kafkaLog.read(readOffset, 5 * 1024 * 1024, Option.<Object>apply(maxOffset)).messageSet();
        final Iterator<MessageAndOffset> iterator = messageSet.iterator();
        long firstOffset = Long.MIN_VALUE;
        long lastOffset = Long.MIN_VALUE;
        long totalBytes = 0;
        while (iterator.hasNext()) {
            final MessageAndOffset messageAndOffset = iterator.next();
            if (firstOffset == Long.MIN_VALUE)
                firstOffset = messageAndOffset.offset();
            // always remember the last seen offset for debug purposes below
            lastOffset = messageAndOffset.offset();
            final byte[] payloadBytes = ByteBufferUtils.readBytes(messageAndOffset.message().payload());
            if (LOG.isTraceEnabled()) {
                final byte[] keyBytes = ByteBufferUtils.readBytes(messageAndOffset.message().key());
                LOG.trace("Read message {} contains {}", bytesToHex(keyBytes), bytesToHex(payloadBytes));
            }
            totalBytes += payloadBytes.length;
            messages.add(new JournalReadEntry(payloadBytes, messageAndOffset.offset()));
            // remember where to read from
            nextReadOffset = messageAndOffset.nextOffset();
        }
        if (messages.isEmpty()) {
            LOG.debug("No messages available to read for offset interval [{}, {}).", readOffset, maxOffset);
        } else {
            LOG.debug("Read {} messages, total payload size {}, from journal, offset interval [{}, {}], requested read at {}", messages.size(), totalBytes, firstOffset, lastOffset, readOffset);
        }
    } catch (OffsetOutOfRangeException e) {
        // This is fine, the reader tries to read faster than the writer committed data. Next read will get the data.
        LOG.debug("Offset out of range, no messages available starting at offset {}", readOffset);
    } catch (Exception e) {
        // sigh.
        if (shuttingDown) {
            LOG.debug("Caught exception during shutdown, ignoring it because we might have been blocked on a read.");
            return Collections.emptyList();
        }
        //noinspection ConstantConditions
        if (e instanceof ClosedByInterruptException) {
            LOG.debug("Interrupted while reading from journal, during shutdown this is harmless and ignored.", e);
        } else {
            throw e;
        }
    }
    readMessages.mark(messages.size());
    return messages;
}