Examples with EventProcessorException org.graylog.events.processor.EventProcessorException used on opensource projects

Example 1 with EventProcessorException

use of org.graylog.events.processor.EventProcessorException in project graylog2-server by Graylog2.

the class PivotAggregationSearch method extractValues.

@VisibleForTesting
ImmutableList<AggregationKeyResult> extractValues(PivotResult pivotResult) throws EventProcessorException {
    final ImmutableList.Builder<AggregationKeyResult> results = ImmutableList.builder();
    // }
    for (final PivotResult.Row row : pivotResult.rows()) {
        if (!"leaf".equals(row.source())) {
            // "non-leaf" values can show up when the "rollup" feature is enabled in the pivot search type
            continue;
        }
        // Safety guard against programming errors
        if (row.key().size() == 0 || isNullOrEmpty(row.key().get(0))) {
            throw new EventProcessorException("Invalid row key! Expected at least the date range timestamp value: " + row.key().toString(), true, eventDefinition);
        }
        // We always wrap aggregations in date range buckets so we can run aggregations for multiple ranges at once.
        // The timestamp value of the date range bucket will be part of the result.
        final String timeKey = row.key().get(0);
        final ImmutableList<String> groupKey;
        if (row.key().size() > 1) {
            // The date range bucket value must not be exposed to consumers as part of the key so they
            // don't have to unwrap the key all the time.
            groupKey = row.key().subList(1, row.key().size());
        } else {
            groupKey = ImmutableList.of();
        }
        final ImmutableList.Builder<AggregationSeriesValue> values = ImmutableList.builder();
        for (final PivotResult.Value value : row.values()) {
            if (!"row-leaf".equals(value.source())) {
                // "row-inner" values can show up when the "rollup" feature is enabled in the pivot search type
                continue;
            }
            for (final AggregationSeries series : config.series()) {
                if (!value.key().isEmpty() && value.key().get(0).equals(metricName(series))) {
                    // Some Elasticsearch aggregations can return a "null" value. (e.g. avg on a non-existent field)
                    // We are using NaN in that case to make sure our conditions will work.
                    final Object maybeNumberValue = firstNonNull(value.value(), Double.NaN);
                    if (maybeNumberValue instanceof Number) {
                        final double numberValue = ((Number) maybeNumberValue).doubleValue();
                        final AggregationSeriesValue seriesValue = AggregationSeriesValue.builder().key(groupKey).value(numberValue).series(series).build();
                        values.add(seriesValue);
                    } else {
                        // Should not happen
                        throw new IllegalStateException("Got unexpected non-number value for " + series.toString() + " " + row.toString() + " " + value.toString());
                    }
                }
            }
        }
        results.add(AggregationKeyResult.builder().key(groupKey).timestamp(DateTime.parse(timeKey).withZone(DateTimeZone.UTC)).seriesValues(values.build()).build());
    }
    return results.build();
}

Example 2 with EventProcessorException

use of org.graylog.events.processor.EventProcessorException in project graylog2-server by Graylog2.

the class PivotAggregationSearch method getSearchJob.

private SearchJob getSearchJob(AggregationEventProcessorParameters parameters, String username, long searchWithinMs, long executeEveryMs) throws EventProcessorException {
    Search search = Search.builder().queries(ImmutableSet.of(getAggregationQuery(parameters, searchWithinMs, executeEveryMs), getSourceStreamsQuery(parameters))).parameters(config.queryParameters()).build();
    // This adds all streams if none were provided
    // TODO: Once we introduce "EventProcessor owners" this should only load the permitted streams of the
    // user who created this EventProcessor.
    search = search.addStreamsToQueriesWithoutStreams(() -> permittedStreams.load((streamId) -> true));
    final SearchJob searchJob = queryEngine.execute(searchJobService.create(search, username));
    try {
        Uninterruptibles.getUninterruptibly(searchJob.getResultFuture(), configurationProvider.get().eventsSearchTimeout(), TimeUnit.MILLISECONDS);
    } catch (ExecutionException e) {
        throw new EventProcessorException("Error executing search job: " + e.getMessage(), false, eventDefinition, e);
    } catch (TimeoutException e) {
        throw new EventProcessorException("Timeout while executing search job.", false, eventDefinition, e);
    } catch (Exception e) {
        throw new EventProcessorException("Unhandled exception in search job.", false, eventDefinition, e);
    }
    return searchJob;
}

Example 3 with EventProcessorException

use of org.graylog.events.processor.EventProcessorException in project graylog2-server by Graylog2.

the class AggregationEventProcessor method sourceMessagesForEvent.

@Override
public void sourceMessagesForEvent(Event event, Consumer<List<MessageSummary>> messageConsumer, long limit) throws EventProcessorException {
    if (config.series().isEmpty()) {
        if (limit <= 0) {
            return;
        }
        final EventOriginContext.ESEventOriginContext esContext = EventOriginContext.parseESContext(event.getOriginContext()).orElseThrow(() -> new EventProcessorException("Failed to parse origin context", false, eventDefinition));
        try {
            final ResultMessage message;
            message = messages.get(esContext.messageId(), esContext.indexName());
            messageConsumer.accept(Lists.newArrayList(new MessageSummary(message.getIndex(), message.getMessage())));
        } catch (IOException e) {
            throw new EventProcessorException("Failed to query origin context message", false, eventDefinition, e);
        }
    } else {
        final AtomicLong msgCount = new AtomicLong(0L);
        final MoreSearch.ScrollCallback callback = (messages, continueScrolling) -> {
            final List<MessageSummary> summaries = Lists.newArrayList();
            for (final ResultMessage resultMessage : messages) {
                if (msgCount.incrementAndGet() > limit) {
                    continueScrolling.set(false);
                    break;
                }
                final Message msg = resultMessage.getMessage();
                summaries.add(new MessageSummary(resultMessage.getIndex(), msg));
            }
            messageConsumer.accept(summaries);
        };
        ElasticsearchQueryString scrollQueryString = ElasticsearchQueryString.of(config.query());
        scrollQueryString = scrollQueryString.concatenate(groupByQueryString(event));
        LOG.debug("scrollQueryString: {}", scrollQueryString);
        final TimeRange timeRange = AbsoluteRange.create(event.getTimerangeStart(), event.getTimerangeEnd());
        moreSearch.scrollQuery(scrollQueryString.queryString(), config.streams(), config.queryParameters(), timeRange, Math.min(500, Ints.saturatedCast(limit)), callback);
    }
}

Example 4 with EventProcessorException

use of org.graylog.events.processor.EventProcessorException in project graylog2-server by Graylog2.

the class MoreSearchAdapterES6 method scrollEvents.

@Override
public void scrollEvents(String queryString, TimeRange timeRange, Set<String> affectedIndices, Set<String> streams, String scrollTime, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException {
    final QueryBuilder query = (queryString.trim().isEmpty() || queryString.trim().equals("*")) ? matchAllQuery() : queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
    final BoolQueryBuilder filter = boolQuery().filter(query).filter(requireNonNull(TimeRangeQueryFactory.create(timeRange)));
    // Filtering with an empty streams list doesn't work and would return zero results
    if (!streams.isEmpty()) {
        filter.filter(termsQuery(Message.FIELD_STREAMS, streams));
    }
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filter).size(batchSize);
    final Search.Builder searchBuilder = new Search.Builder(searchSourceBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(affectedIndices.isEmpty() ? Collections.singleton("") : affectedIndices).addSort(new Sort("timestamp", Sort.Sorting.ASC)).allowNoIndices(false).ignoreUnavailable(false).setParameter(Parameters.SCROLL, scrollTime);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Query:\n{}", searchSourceBuilder.toString(new ToXContent.MapParams(Collections.singletonMap("pretty", "true"))));
        LOG.debug("Execute search: {}", searchBuilder.build().toString());
    }
    final ScrollResult scrollResult = scroll.scroll(searchBuilder.build(), () -> "Unable to scroll indices.", searchSourceBuilder.toString(), scrollTime, Collections.emptyList());
    final AtomicBoolean continueScrolling = new AtomicBoolean(true);
    final Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        ScrollResult.ScrollChunk scrollChunk = scrollResult.nextChunk();
        while (continueScrolling.get() && scrollChunk != null) {
            final List<ResultMessage> messages = scrollChunk.getMessages();
            LOG.debug("Passing <{}> messages to callback", messages.size());
            resultCallback.accept(Collections.unmodifiableList(messages), continueScrolling);
            // Stop if the resultCallback told us to stop
            if (!continueScrolling.get()) {
                break;
            }
            scrollChunk = scrollResult.nextChunk();
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        try {
            // Tell Elasticsearch that we are done with the scroll so it can release resources as soon as possible
            // instead of waiting for the scroll timeout to kick in.
            scrollResult.cancel();
        } catch (Exception ignored) {
        }
        LOG.debug("Scrolling done - took {} ms", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
    }
}

Example 5 with EventProcessorException

use of org.graylog.events.processor.EventProcessorException in project graylog2-server by Graylog2.

the class MoreSearchAdapterES7 method scrollEvents.

@Override
public void scrollEvents(String queryString, TimeRange timeRange, Set<String> affectedIndices, Set<String> streams, String scrollTime, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException {
    final ScrollCommand scrollCommand = buildScrollCommand(queryString, timeRange, affectedIndices, streams, batchSize);
    final ScrollResult scrollResult = scroll.scroll(scrollCommand);
    final AtomicBoolean continueScrolling = new AtomicBoolean(true);
    final Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        ScrollResult.ScrollChunk scrollChunk = scrollResult.nextChunk();
        while (continueScrolling.get() && scrollChunk != null) {
            final List<ResultMessage> messages = scrollChunk.getMessages();
            LOG.debug("Passing <{}> messages to callback", messages.size());
            resultCallback.accept(Collections.unmodifiableList(messages), continueScrolling);
            // Stop if the resultCallback told us to stop
            if (!continueScrolling.get()) {
                break;
            }
            scrollChunk = scrollResult.nextChunk();
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        try {
            // Tell Elasticsearch that we are done with the scroll so it can release resources as soon as possible
            // instead of waiting for the scroll timeout to kick in.
            scrollResult.cancel();
        } catch (Exception ignored) {
        }
        LOG.debug("Scrolling done - took {} ms", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
    }
}