Search in sources :

Example 6 with ESSearchTypeHandler

use of org.graylog.storage.elasticsearch6.views.searchtypes.ESSearchTypeHandler in project graylog2-server by Graylog2.

the class ElasticsearchBackend method doRun.

@Override
public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {
    if (query.searchTypes().isEmpty()) {
        return QueryResult.builder().query(query).searchTypes(Collections.emptyMap()).errors(new HashSet<>(queryContext.errors())).build();
    }
    LOG.debug("Running query {} for job {}", query.id(), job.getId());
    final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
    final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
    final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
    final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());
    final List<SearchRequest> searches = searchTypeIds.stream().map(searchTypeId -> {
        final Set<String> affectedIndicesForSearchType = query.searchTypes().stream().filter(s -> s.id().equalsIgnoreCase(searchTypeId)).findFirst().flatMap(searchType -> {
            if (searchType.effectiveStreams().isEmpty() && !query.globalOverride().flatMap(GlobalOverride::timerange).isPresent() && !searchType.timerange().isPresent()) {
                return Optional.empty();
            }
            final Set<String> usedStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
            return Optional.of(indexLookup.indexNamesForStreamsInTimeRange(usedStreamIds, query.effectiveTimeRange(searchType)));
        }).orElse(affectedIndices);
        Set<String> indices = affectedIndicesForSearchType.isEmpty() ? Collections.singleton("") : affectedIndicesForSearchType;
        return new SearchRequest().source(searchTypeQueries.get(searchTypeId)).indices(indices.toArray(new String[0])).indicesOptions(IndicesOptions.fromOptions(false, false, true, false));
    }).collect(Collectors.toList());
    final List<MultiSearchResponse.Item> results = client.msearch(searches, "Unable to perform search query: ");
    for (SearchType searchType : query.searchTypes()) {
        final String searchTypeId = searchType.id();
        final Provider<ESSearchTypeHandler<? extends SearchType>> handlerProvider = elasticsearchSearchTypeHandlers.get(searchType.type());
        if (handlerProvider == null) {
            LOG.error("Unknown search type '{}', cannot convert query result.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        if (isSearchTypeWithError(queryContext, searchTypeId)) {
            LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        // we create a new instance because some search type handlers might need to track information between generating the query and
        // processing its result, such as aggregations, which depend on the name and type
        final ESSearchTypeHandler<? extends SearchType> handler = handlerProvider.get();
        final int searchTypeIndex = searchTypeIds.indexOf(searchTypeId);
        final MultiSearchResponse.Item multiSearchResponse = results.get(searchTypeIndex);
        if (multiSearchResponse.isFailure()) {
            ElasticsearchException e = new ElasticsearchException("Search type returned error: ", multiSearchResponse.getFailure());
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else if (checkForFailedShards(multiSearchResponse).isPresent()) {
            ElasticsearchException e = checkForFailedShards(multiSearchResponse).get();
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else {
            final SearchType.Result searchTypeResult = handler.extractResult(job, query, searchType, multiSearchResponse.getResponse(), queryContext);
            if (searchTypeResult != null) {
                resultsMap.put(searchTypeId, searchTypeResult);
            }
        }
    }
    LOG.debug("Query {} ran for job {}", query.id(), job.getId());
    return QueryResult.builder().query(query).searchTypes(resultsMap).errors(new HashSet<>(queryContext.errors())).build();
}
Also used : ESSearchTypeHandler(org.graylog.storage.elasticsearch7.views.searchtypes.ESSearchTypeHandler) AndFilter(org.graylog.plugins.views.search.filter.AndFilter) ElasticsearchClient(org.graylog.storage.elasticsearch7.ElasticsearchClient) Arrays(java.util.Arrays) BackendQuery(org.graylog.plugins.views.search.engine.BackendQuery) QueryBackend(org.graylog.plugins.views.search.engine.QueryBackend) Provider(javax.inject.Provider) LoggerFactory(org.slf4j.LoggerFactory) FieldTypeException(org.graylog2.indexer.FieldTypeException) MultiSearchResponse(org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.MultiSearchResponse) StreamFilter(org.graylog.plugins.views.search.filter.StreamFilter) Map(java.util.Map) IndicesOptions(org.graylog.shaded.elasticsearch7.org.elasticsearch.action.support.IndicesOptions) SearchConfig(org.graylog.plugins.views.search.engine.SearchConfig) Set(java.util.Set) Collectors(java.util.stream.Collectors) BoolQueryBuilder(org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.BoolQueryBuilder) Objects(java.util.Objects) List(java.util.List) Filter(org.graylog.plugins.views.search.Filter) Optional(java.util.Optional) SearchResponse(org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchResponse) Query(org.graylog.plugins.views.search.Query) SearchTypeErrorParser(org.graylog.plugins.views.search.errors.SearchTypeErrorParser) HashMap(java.util.HashMap) ElasticsearchException(org.graylog2.indexer.ElasticsearchException) ShardOperationFailedException(org.graylog.shaded.elasticsearch7.org.elasticsearch.action.ShardOperationFailedException) ArrayList(java.util.ArrayList) GlobalOverride(org.graylog.plugins.views.search.GlobalOverride) Inject(javax.inject.Inject) HashSet(java.util.HashSet) OrFilter(org.graylog.plugins.views.search.filter.OrFilter) SearchType(org.graylog.plugins.views.search.SearchType) QueryStringFilter(org.graylog.plugins.views.search.filter.QueryStringFilter) SearchTypeError(org.graylog.plugins.views.search.errors.SearchTypeError) QueryResult(org.graylog.plugins.views.search.QueryResult) TimeRangeQueryFactory(org.graylog.storage.elasticsearch7.TimeRangeQueryFactory) SearchJob(org.graylog.plugins.views.search.SearchJob) QueryBuilder(org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilder) SearchSourceBuilder(org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder) Logger(org.slf4j.Logger) SearchRequest(org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchRequest) Maps(com.google.common.collect.Maps) QueryStringDecorators(org.graylog.plugins.views.search.elasticsearch.QueryStringDecorators) Named(com.google.inject.name.Named) IndexLookup(org.graylog.plugins.views.search.elasticsearch.IndexLookup) QueryBuilders(org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilders) Message(org.graylog2.plugin.Message) Collections(java.util.Collections) SearchRequest(org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchRequest) Set(java.util.Set) HashSet(java.util.HashSet) ArrayList(java.util.ArrayList) ElasticsearchException(org.graylog2.indexer.ElasticsearchException) QueryResult(org.graylog.plugins.views.search.QueryResult) SearchSourceBuilder(org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder) GlobalOverride(org.graylog.plugins.views.search.GlobalOverride) MultiSearchResponse(org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.MultiSearchResponse) ESSearchTypeHandler(org.graylog.storage.elasticsearch7.views.searchtypes.ESSearchTypeHandler) SearchType(org.graylog.plugins.views.search.SearchType) HashSet(java.util.HashSet) GlobalOverride(org.graylog.plugins.views.search.GlobalOverride)

Aggregations

SearchType (org.graylog.plugins.views.search.SearchType)6 Maps (com.google.common.collect.Maps)4 Map (java.util.Map)4 Provider (javax.inject.Provider)4 GlobalOverride (org.graylog.plugins.views.search.GlobalOverride)4 Query (org.graylog.plugins.views.search.Query)4 QueryResult (org.graylog.plugins.views.search.QueryResult)4 SearchJob (org.graylog.plugins.views.search.SearchJob)4 IndexLookup (org.graylog.plugins.views.search.elasticsearch.IndexLookup)4 QueryStringDecorators (org.graylog.plugins.views.search.elasticsearch.QueryStringDecorators)4 BackendQuery (org.graylog.plugins.views.search.engine.BackendQuery)4 SearchConfig (org.graylog.plugins.views.search.engine.SearchConfig)4 SearchTypeError (org.graylog.plugins.views.search.errors.SearchTypeError)4 ESSearchTypeHandler (org.graylog.storage.elasticsearch6.views.searchtypes.ESSearchTypeHandler)3 ESSearchTypeHandler (org.graylog.storage.elasticsearch7.views.searchtypes.ESSearchTypeHandler)3 ImmutableSet (com.google.common.collect.ImmutableSet)2 Named (com.google.inject.name.Named)2 ArrayList (java.util.ArrayList)2 Collections (java.util.Collections)2 HashMap (java.util.HashMap)2