Examples with Messages org.graylog2.indexer.messages.Messages used on opensource projects

Example 21 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class FieldContentValueAlertCondition method runCheck.

@Override
public CheckResult runCheck() {
    String filter = buildQueryFilter(stream.getId(), query);
    String query = field + ":\"" + value + "\"";
    Integer backlogSize = getBacklog();
    boolean backlogEnabled = false;
    int searchLimit = 1;
    if (backlogSize != null && backlogSize > 0) {
        backlogEnabled = true;
        searchLimit = backlogSize;
    }
    try {
        SearchResult result = searches.search(query, filter, RelativeRange.create(configuration.getAlertCheckInterval()), searchLimit, 0, new Sorting(Message.FIELD_TIMESTAMP, Sorting.Direction.DESC));
        final List<MessageSummary> summaries;
        if (backlogEnabled) {
            summaries = Lists.newArrayListWithCapacity(result.getResults().size());
            for (ResultMessage resultMessage : result.getResults()) {
                final Message msg = resultMessage.getMessage();
                summaries.add(new MessageSummary(resultMessage.getIndex(), msg));
            }
        } else {
            summaries = Collections.emptyList();
        }
        final long count = result.getTotalResults();
        final String resultDescription = "Stream received messages matching <" + query + "> " + "(Current grace time: " + grace + " minutes)";
        if (count > 0) {
            LOG.debug("Alert check <{}> found [{}] messages.", id, count);
            return new CheckResult(true, this, resultDescription, Tools.nowUTC(), summaries);
        } else {
            LOG.debug("Alert check <{}> returned no results.", id);
            return new NegativeCheckResult();
        }
    } catch (InvalidRangeParametersException e) {
        // cannot happen lol
        LOG.error("Invalid timerange.", e);
        return null;
    }
}

Example 22 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class MessageCountAlertCondition method runCheck.

@Override
public CheckResult runCheck() {
    try {
        // Create an absolute range from the relative range to make sure it doesn't change during the two
        // search requests. (count and find messages)
        // This is needed because the RelativeRange computes the range from NOW on every invocation of getFrom() and
        // getTo().
        // See: https://github.com/Graylog2/graylog2-server/issues/2382
        final RelativeRange relativeRange = RelativeRange.create(time * 60);
        final AbsoluteRange range = AbsoluteRange.create(relativeRange.getFrom(), relativeRange.getTo());
        final String filter = buildQueryFilter(stream.getId(), query);
        final CountResult result = searches.count("*", range, filter);
        final long count = result.count();
        LOG.debug("Alert check <{}> result: [{}]", id, count);
        final boolean triggered;
        switch(thresholdType) {
            case MORE:
                triggered = count > threshold;
                break;
            case LESS:
                triggered = count < threshold;
                break;
            default:
                triggered = false;
        }
        if (triggered) {
            final List<MessageSummary> summaries = Lists.newArrayList();
            if (getBacklog() > 0) {
                final SearchResult backlogResult = searches.search("*", filter, range, getBacklog(), 0, new Sorting(Message.FIELD_TIMESTAMP, Sorting.Direction.DESC));
                for (ResultMessage resultMessage : backlogResult.getResults()) {
                    final Message msg = resultMessage.getMessage();
                    summaries.add(new MessageSummary(resultMessage.getIndex(), msg));
                }
            }
            final String resultDescription = "Stream had " + count + " messages in the last " + time + " minutes with trigger condition " + thresholdType.toString().toLowerCase(Locale.ENGLISH) + " than " + threshold + " messages. " + "(Current grace time: " + grace + " minutes)";
            return new CheckResult(true, this, resultDescription, Tools.nowUTC(), summaries);
        } else {
            return new NegativeCheckResult();
        }
    } catch (InvalidRangeParametersException e) {
        // cannot happen lol
        LOG.error("Invalid timerange.", e);
        return null;
    }
}

Example 23 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class MessageFilterChainProcessor method process.

@Override
public Messages process(Messages messages) {
    for (final MessageFilter filter : filterRegistry) {
        for (Message msg : messages) {
            final String timerName = name(filter.getClass(), "executionTime");
            final Timer timer = metricRegistry.timer(timerName);
            final Timer.Context timerContext = timer.time();
            try {
                LOG.debug("Applying filter [{}] on message <{}>.", filter.getName(), msg.getId());
                if (filter.filter(msg)) {
                    LOG.debug("Filter [{}] marked message <{}> to be discarded. Dropping message.", filter.getName(), msg.getId());
                    msg.setFilterOut(true);
                    filteredOutMessages.mark();
                    messageQueueAcknowledger.acknowledge(msg);
                }
            } catch (Exception e) {
                final String shortError = String.format(Locale.US, "Could not apply filter [%s] on message <%s>", filter.getName(), msg.getId());
                if (LOG.isDebugEnabled()) {
                    LOG.error("{}:", shortError, e);
                } else {
                    LOG.error("{}:\n{}", shortError, ExceptionUtils.getShortenedStackTrace(e));
                }
                msg.addProcessingError(new Message.ProcessingError(ProcessingFailureCause.MessageFilterException, shortError, ExceptionUtils.getRootCauseMessage(e)));
            } finally {
                final long elapsedNanos = timerContext.stop();
                msg.recordTiming(serverStatus, timerName, elapsedNanos);
            }
        }
    }
    return messages;
}

Example 24 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class IndexRangesResource method rebuildIndexSet.

@POST
@Timed
@Path("/index_set/{indexSetId}/rebuild")
@RequiresPermissions(RestPermissions.INDEXRANGES_REBUILD)
@ApiOperation(value = "Rebuild/sync index range information for the given index set.", notes = "This triggers a systemjob that scans every index in the given index set and stores meta information " + "about what indices contain messages in what timeranges. It atomically overwrites " + "already existing meta information.")
@ApiResponses(value = { @ApiResponse(code = 202, message = "Rebuild/sync systemjob triggered.") })
@Produces(MediaType.APPLICATION_JSON)
@AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE_JOB)
public Response rebuildIndexSet(@ApiParam(name = "indexSetId") @PathParam("indexSetId") @NotBlank final String indexSetId) {
    final IndexSet indexSet = indexSetRegistry.get(indexSetId).orElseThrow(() -> new javax.ws.rs.NotFoundException("Index set <" + indexSetId + "> not found!"));
    submitIndexRangesJob(Collections.singleton(indexSet));
    return Response.accepted().build();
}

Example 25 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class MessagesAdapterES6 method bulkIndexChunked.

private List<Messages.IndexingError> bulkIndexChunked(ChunkedBulkIndexer.Chunk command) throws ChunkedBulkIndexer.EntityTooLargeException, IOException {
    final List<IndexingRequest> messageList = command.requests;
    final int offset = command.offset;
    int chunkSize = Math.min(messageList.size(), command.size);
    final List<BulkResult.BulkResultItem> failedItems = new ArrayList<>();
    final Iterable<List<IndexingRequest>> chunks = Iterables.partition(messageList.subList(offset, messageList.size()), chunkSize);
    int chunkCount = 1;
    int indexedSuccessfully = 0;
    for (List<IndexingRequest> chunk : chunks) {
        final BulkResult result = bulkIndexChunk(chunk);
        if (result.getResponseCode() == 413) {
            throw new ChunkedBulkIndexer.EntityTooLargeException(indexedSuccessfully, indexingErrorsFrom(failedItems, messageList));
        }
        if (result.getResponseCode() >= 400) {
            throw JestUtils.specificException(() -> "Error during bulk indexing: ", result.getJsonObject().get("error"));
        }
        indexedSuccessfully += chunk.size();
        final List<BulkResult.BulkResultItem> remainingFailures = result.getFailedItems();
        failedItems.addAll(remainingFailures);
        if (LOG.isDebugEnabled()) {
            String chunkInfo = "";
            if (chunkSize != messageList.size()) {
                chunkInfo = String.format(Locale.ROOT, " (chunk %d/%d offset %d)", chunkCount, (int) Math.ceil((double) messageList.size() / chunkSize), offset);
            }
            LOG.debug("Index: Bulk indexed {} messages{}, failures: {}", result.getItems().size(), chunkInfo, failedItems.size());
        }
        if (!remainingFailures.isEmpty()) {
            LOG.error("Failed to index [{}] messages. Please check the index error log in your web interface for the reason. Error: {}", remainingFailures.size(), result.getErrorMessage());
        }
        chunkCount++;
    }
    return indexingErrorsFrom(failedItems, messageList);
}