use of org.graylog.storage.elasticsearch7.views.ESGeneratedQueryContext in project graylog2-server by Graylog2.
the class ESMessageList method doExtractResult.
@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, MessageList searchType, SearchResult result, MetricAggregation aggregations, ESGeneratedQueryContext queryContext) {
// noinspection unchecked
final List<ResultMessageSummary> messages = result.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source, hit.highlight)).map((resultMessage) -> ResultMessageSummary.create(resultMessage.highlightRanges, resultMessage.getMessage().getFields(), resultMessage.getIndex())).collect(Collectors.toList());
final String undecoratedQueryString = query.query().queryString();
final String queryString = this.esQueryDecorators.decorate(undecoratedQueryString, job, query);
final DateTime from = query.effectiveTimeRange(searchType).getFrom();
final DateTime to = query.effectiveTimeRange(searchType).getTo();
final SearchResponse searchResponse = SearchResponse.create(undecoratedQueryString, queryString, Collections.emptySet(), messages, Collections.emptySet(), 0, result.getTotal(), from, to);
final SearchResponse decoratedSearchResponse = decoratorProcessor.decorateSearchResponse(searchResponse, searchType.decorators());
final MessageList.Result.Builder resultBuilder = MessageList.Result.result(searchType.id()).messages(decoratedSearchResponse.messages()).effectiveTimerange(AbsoluteRange.create(from, to)).totalResults(decoratedSearchResponse.totalResults());
return searchType.name().map(resultBuilder::name).orElse(resultBuilder).build();
}
use of org.graylog.storage.elasticsearch7.views.ESGeneratedQueryContext in project graylog2-server by Graylog2.
the class ESMessageList method doExtractResult.
@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, MessageList searchType, org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchResponse result, Aggregations aggregations, ESGeneratedQueryContext queryContext) {
final List<ResultMessageSummary> messages = StreamSupport.stream(result.getHits().spliterator(), false).map(ESMessageList::resultMessageFromSearchHit).map((resultMessage) -> ResultMessageSummary.create(resultMessage.highlightRanges, resultMessage.getMessage().getFields(), resultMessage.getIndex())).collect(Collectors.toList());
final String undecoratedQueryString = query.query().queryString();
final String queryString = this.esQueryDecorators.decorate(undecoratedQueryString, job, query);
final DateTime from = query.effectiveTimeRange(searchType).getFrom();
final DateTime to = query.effectiveTimeRange(searchType).getTo();
final SearchResponse searchResponse = SearchResponse.create(undecoratedQueryString, queryString, Collections.emptySet(), messages, Collections.emptySet(), 0, result.getHits().getTotalHits().value, from, to);
final SearchResponse decoratedSearchResponse = decoratorProcessor.decorateSearchResponse(searchResponse, searchType.decorators());
final MessageList.Result.Builder resultBuilder = MessageList.Result.result(searchType.id()).messages(decoratedSearchResponse.messages()).effectiveTimerange(AbsoluteRange.create(from, to)).totalResults(decoratedSearchResponse.totalResults());
return searchType.name().map(resultBuilder::name).orElse(resultBuilder).build();
}
use of org.graylog.storage.elasticsearch7.views.ESGeneratedQueryContext in project graylog2-server by Graylog2.
the class ESPivot method processSeries.
private void processSeries(PivotResult.Row.Builder rowBuilder, SearchResponse searchResult, ESGeneratedQueryContext queryContext, Pivot pivot, ArrayDeque<String> columnKeys, HasAggregations aggregation, boolean rollup, String source) {
pivot.series().forEach(seriesSpec -> {
final ESPivotSeriesSpecHandler<? extends SeriesSpec, ? extends Aggregation> seriesHandler = seriesHandlers.get(seriesSpec.type());
final Aggregation series = seriesHandler.extractAggregationFromResult(pivot, seriesSpec, aggregation, queryContext);
seriesHandler.handleResult(pivot, seriesSpec, searchResult, series, this, queryContext).map(value -> {
columnKeys.addLast(value.id());
final PivotResult.Value v = PivotResult.Value.create(columnKeys, value.value(), rollup, source);
columnKeys.removeLast();
return v;
}).forEach(rowBuilder::addValue);
});
}
use of org.graylog.storage.elasticsearch7.views.ESGeneratedQueryContext in project graylog2-server by Graylog2.
the class ElasticsearchBackend method generate.
@Override
public ESGeneratedQueryContext generate(SearchJob job, Query query, SearchConfig searchConfig) {
final BackendQuery backendQuery = query.query();
validateQueryTimeRange(query, searchConfig);
final Set<SearchType> searchTypes = query.searchTypes();
final String queryString = this.queryStringDecorators.decorate(backendQuery.queryString(), job, query);
final QueryBuilder normalizedRootQuery = normalizeQueryString(queryString);
final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().filter(normalizedRootQuery);
// add the optional root query filters
generateFilterClause(query.filter(), job, query).map(boolQuery::filter);
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(boolQuery).from(0).size(0).trackTotalHits(true);
final ESGeneratedQueryContext queryContext = queryContextFactory.create(this, searchSourceBuilder, job, query);
for (SearchType searchType : searchTypes) {
final Optional<SearchTypeError> searchTypeError = validateSearchType(query, searchType, searchConfig);
if (searchTypeError.isPresent()) {
LOG.error("Invalid search type {} for elasticsearch backend, cannot generate query part. Skipping this search type.", searchType.type());
queryContext.addError(searchTypeError.get());
continue;
}
final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType);
final Set<String> effectiveStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery().must(searchTypeSourceBuilder.query()).must(Objects.requireNonNull(TimeRangeQueryFactory.create(query.effectiveTimeRange(searchType)), "Timerange for search type " + searchType.id() + " cannot be found in query or search type.")).must(QueryBuilders.termsQuery(Message.FIELD_STREAMS, effectiveStreamIds));
searchType.query().ifPresent(searchTypeQuery -> {
final String searchTypeQueryString = this.queryStringDecorators.decorate(searchTypeQuery.queryString(), job, query);
final QueryBuilder normalizedSearchTypeQuery = normalizeQueryString(searchTypeQueryString);
searchTypeOverrides.must(normalizedSearchTypeQuery);
});
searchTypeSourceBuilder.query(searchTypeOverrides);
final String type = searchType.type();
final Provider<ESSearchTypeHandler<? extends SearchType>> searchTypeHandler = elasticsearchSearchTypeHandlers.get(type);
if (searchTypeHandler == null) {
LOG.error("Unknown search type {} for elasticsearch backend, cannot generate query part. Skipping this search type.", type);
queryContext.addError(new SearchTypeError(query, searchType.id(), "Unknown search type '" + type + "' for elasticsearch backend, cannot generate query"));
continue;
}
if (isSearchTypeWithError(queryContext, searchType.id())) {
LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
// no need to add another error here, as the query generation code will have added the error about the missing handler already
continue;
}
searchTypeHandler.get().generateQueryPart(job, query, searchType, queryContext);
}
return queryContext;
}
use of org.graylog.storage.elasticsearch7.views.ESGeneratedQueryContext in project graylog2-server by Graylog2.
the class ElasticsearchBackend method doRun.
@Override
public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {
if (query.searchTypes().isEmpty()) {
return QueryResult.builder().query(query).searchTypes(Collections.emptyMap()).errors(new HashSet<>(queryContext.errors())).build();
}
LOG.debug("Running query {} for job {}", query.id(), job.getId());
final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());
final List<SearchRequest> searches = searchTypeIds.stream().map(searchTypeId -> {
final Set<String> affectedIndicesForSearchType = query.searchTypes().stream().filter(s -> s.id().equalsIgnoreCase(searchTypeId)).findFirst().flatMap(searchType -> {
if (searchType.effectiveStreams().isEmpty() && !query.globalOverride().flatMap(GlobalOverride::timerange).isPresent() && !searchType.timerange().isPresent()) {
return Optional.empty();
}
final Set<String> usedStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
return Optional.of(indexLookup.indexNamesForStreamsInTimeRange(usedStreamIds, query.effectiveTimeRange(searchType)));
}).orElse(affectedIndices);
Set<String> indices = affectedIndicesForSearchType.isEmpty() ? Collections.singleton("") : affectedIndicesForSearchType;
return new SearchRequest().source(searchTypeQueries.get(searchTypeId)).indices(indices.toArray(new String[0])).indicesOptions(IndicesOptions.fromOptions(false, false, true, false));
}).collect(Collectors.toList());
final List<MultiSearchResponse.Item> results = client.msearch(searches, "Unable to perform search query: ");
for (SearchType searchType : query.searchTypes()) {
final String searchTypeId = searchType.id();
final Provider<ESSearchTypeHandler<? extends SearchType>> handlerProvider = elasticsearchSearchTypeHandlers.get(searchType.type());
if (handlerProvider == null) {
LOG.error("Unknown search type '{}', cannot convert query result.", searchType.type());
// no need to add another error here, as the query generation code will have added the error about the missing handler already
continue;
}
if (isSearchTypeWithError(queryContext, searchTypeId)) {
LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
// no need to add another error here, as the query generation code will have added the error about the missing handler already
continue;
}
// we create a new instance because some search type handlers might need to track information between generating the query and
// processing its result, such as aggregations, which depend on the name and type
final ESSearchTypeHandler<? extends SearchType> handler = handlerProvider.get();
final int searchTypeIndex = searchTypeIds.indexOf(searchTypeId);
final MultiSearchResponse.Item multiSearchResponse = results.get(searchTypeIndex);
if (multiSearchResponse.isFailure()) {
ElasticsearchException e = new ElasticsearchException("Search type returned error: ", multiSearchResponse.getFailure());
queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
} else if (checkForFailedShards(multiSearchResponse).isPresent()) {
ElasticsearchException e = checkForFailedShards(multiSearchResponse).get();
queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
} else {
final SearchType.Result searchTypeResult = handler.extractResult(job, query, searchType, multiSearchResponse.getResponse(), queryContext);
if (searchTypeResult != null) {
resultsMap.put(searchTypeId, searchTypeResult);
}
}
}
LOG.debug("Query {} ran for job {}", query.id(), job.getId());
return QueryResult.builder().query(query).searchTypes(resultsMap).errors(new HashSet<>(queryContext.errors())).build();
}
Aggregations