Examples with Messages org.graylog2.indexer.messages.Messages used on opensource projects

Example 86 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class MoreSearchAdapterES6 method scrollEvents.

@Override
public void scrollEvents(String queryString, TimeRange timeRange, Set<String> affectedIndices, Set<String> streams, String scrollTime, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException {
    final QueryBuilder query = (queryString.trim().isEmpty() || queryString.trim().equals("*")) ? matchAllQuery() : queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
    final BoolQueryBuilder filter = boolQuery().filter(query).filter(requireNonNull(TimeRangeQueryFactory.create(timeRange)));
    // Filtering with an empty streams list doesn't work and would return zero results
    if (!streams.isEmpty()) {
        filter.filter(termsQuery(Message.FIELD_STREAMS, streams));
    }
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filter).size(batchSize);
    final Search.Builder searchBuilder = new Search.Builder(searchSourceBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(affectedIndices.isEmpty() ? Collections.singleton("") : affectedIndices).addSort(new Sort("timestamp", Sort.Sorting.ASC)).allowNoIndices(false).ignoreUnavailable(false).setParameter(Parameters.SCROLL, scrollTime);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Query:\n{}", searchSourceBuilder.toString(new ToXContent.MapParams(Collections.singletonMap("pretty", "true"))));
        LOG.debug("Execute search: {}", searchBuilder.build().toString());
    }
    final ScrollResult scrollResult = scroll.scroll(searchBuilder.build(), () -> "Unable to scroll indices.", searchSourceBuilder.toString(), scrollTime, Collections.emptyList());
    final AtomicBoolean continueScrolling = new AtomicBoolean(true);
    final Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        ScrollResult.ScrollChunk scrollChunk = scrollResult.nextChunk();
        while (continueScrolling.get() && scrollChunk != null) {
            final List<ResultMessage> messages = scrollChunk.getMessages();
            LOG.debug("Passing <{}> messages to callback", messages.size());
            resultCallback.accept(Collections.unmodifiableList(messages), continueScrolling);
            // Stop if the resultCallback told us to stop
            if (!continueScrolling.get()) {
                break;
            }
            scrollChunk = scrollResult.nextChunk();
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        try {
            // Tell Elasticsearch that we are done with the scroll so it can release resources as soon as possible
            // instead of waiting for the scroll timeout to kick in.
            scrollResult.cancel();
        } catch (Exception ignored) {
        }
        LOG.debug("Scrolling done - took {} ms", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
    }
}

Example 87 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class ESMessageList method doExtractResult.

@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, MessageList searchType, SearchResult result, MetricAggregation aggregations, ESGeneratedQueryContext queryContext) {
    // noinspection unchecked
    final List<ResultMessageSummary> messages = result.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source, hit.highlight)).map((resultMessage) -> ResultMessageSummary.create(resultMessage.highlightRanges, resultMessage.getMessage().getFields(), resultMessage.getIndex())).collect(Collectors.toList());
    final String undecoratedQueryString = query.query().queryString();
    final String queryString = this.esQueryDecorators.decorate(undecoratedQueryString, job, query);
    final DateTime from = query.effectiveTimeRange(searchType).getFrom();
    final DateTime to = query.effectiveTimeRange(searchType).getTo();
    final SearchResponse searchResponse = SearchResponse.create(undecoratedQueryString, queryString, Collections.emptySet(), messages, Collections.emptySet(), 0, result.getTotal(), from, to);
    final SearchResponse decoratedSearchResponse = decoratorProcessor.decorateSearchResponse(searchResponse, searchType.decorators());
    final MessageList.Result.Builder resultBuilder = MessageList.Result.result(searchType.id()).messages(decoratedSearchResponse.messages()).effectiveTimerange(AbsoluteRange.create(from, to)).totalResults(decoratedSearchResponse.totalResults());
    return searchType.name().map(resultBuilder::name).orElse(resultBuilder).build();
}

Example 88 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class ESMessageList method doExtractResult.

@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, MessageList searchType, org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchResponse result, Aggregations aggregations, ESGeneratedQueryContext queryContext) {
    final List<ResultMessageSummary> messages = StreamSupport.stream(result.getHits().spliterator(), false).map(ESMessageList::resultMessageFromSearchHit).map((resultMessage) -> ResultMessageSummary.create(resultMessage.highlightRanges, resultMessage.getMessage().getFields(), resultMessage.getIndex())).collect(Collectors.toList());
    final String undecoratedQueryString = query.query().queryString();
    final String queryString = this.esQueryDecorators.decorate(undecoratedQueryString, job, query);
    final DateTime from = query.effectiveTimeRange(searchType).getFrom();
    final DateTime to = query.effectiveTimeRange(searchType).getTo();
    final SearchResponse searchResponse = SearchResponse.create(undecoratedQueryString, queryString, Collections.emptySet(), messages, Collections.emptySet(), 0, result.getHits().getTotalHits().value, from, to);
    final SearchResponse decoratedSearchResponse = decoratorProcessor.decorateSearchResponse(searchResponse, searchType.decorators());
    final MessageList.Result.Builder resultBuilder = MessageList.Result.result(searchType.id()).messages(decoratedSearchResponse.messages()).effectiveTimerange(AbsoluteRange.create(from, to)).totalResults(decoratedSearchResponse.totalResults());
    return searchType.name().map(resultBuilder::name).orElse(resultBuilder).build();
}

Example 89 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class MoreSearchAdapterES7 method scrollEvents.

@Override
public void scrollEvents(String queryString, TimeRange timeRange, Set<String> affectedIndices, Set<String> streams, String scrollTime, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException {
    final ScrollCommand scrollCommand = buildScrollCommand(queryString, timeRange, affectedIndices, streams, batchSize);
    final ScrollResult scrollResult = scroll.scroll(scrollCommand);
    final AtomicBoolean continueScrolling = new AtomicBoolean(true);
    final Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        ScrollResult.ScrollChunk scrollChunk = scrollResult.nextChunk();
        while (continueScrolling.get() && scrollChunk != null) {
            final List<ResultMessage> messages = scrollChunk.getMessages();
            LOG.debug("Passing <{}> messages to callback", messages.size());
            resultCallback.accept(Collections.unmodifiableList(messages), continueScrolling);
            // Stop if the resultCallback told us to stop
            if (!continueScrolling.get()) {
                break;
            }
            scrollChunk = scrollResult.nextChunk();
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        try {
            // Tell Elasticsearch that we are done with the scroll so it can release resources as soon as possible
            // instead of waiting for the scroll timeout to kick in.
            scrollResult.cancel();
        } catch (Exception ignored) {
        }
        LOG.debug("Scrolling done - took {} ms", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
    }
}

Example 90 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class EventBacklogService method getMessagesForEvent.

public ImmutableList<MessageSummary> getMessagesForEvent(EventDto eventDto, long backlogSize) throws NotFoundException {
    if (backlogSize <= 0) {
        return ImmutableList.of();
    }
    final EventProcessor.Factory factory = eventProcessorFactories.get(eventDto.eventDefinitionType());
    if (factory == null) {
        throw new NotFoundException("Couldn't find event processor factory for type " + eventDto.eventDefinitionType());
    }
    final EventDefinition eventDefinition = eventDefinitionService.get(eventDto.eventDefinitionId()).orElseThrow(() -> new NotFoundException("Could not find event definintion <" + eventDto.eventDefinitionId() + ">"));
    final EventProcessor eventProcessor = factory.create(eventDefinition);
    final ImmutableList.Builder<MessageSummary> backlogBuilder = ImmutableList.builder();
    try {
        eventProcessor.sourceMessagesForEvent(Event.fromDto(eventDto), backlogBuilder::addAll, backlogSize);
    } catch (EventProcessorException e) {
        // TODO return this error, so it can be included in the notification message?
        LOG.error("Failed to query backlog messages for Event {}", eventDto.id(), e);
    }
    return backlogBuilder.build();
}