Examples with Filter org.graylog.plugins.views.search.Filter used on opensource projects

Example 1 with Filter

use of org.graylog.plugins.views.search.Filter in project graylog2-server by Graylog2.

the class ElasticsearchBackend method generateFilterClause.

// TODO make pluggable
public Optional<QueryBuilder> generateFilterClause(Filter filter, SearchJob job, Query query) {
    if (filter == null) {
        return Optional.empty();
    }
    switch(filter.type()) {
        case AndFilter.NAME:
            final BoolQueryBuilder andBuilder = QueryBuilders.boolQuery();
            filter.filters().stream().map(filter1 -> generateFilterClause(filter1, job, query)).forEach(optQueryBuilder -> optQueryBuilder.ifPresent(andBuilder::must));
            return Optional.of(andBuilder);
        case OrFilter.NAME:
            final BoolQueryBuilder orBuilder = QueryBuilders.boolQuery();
            // TODO for the common case "any of these streams" we can optimize the filter into
            // a single "termsQuery" instead of "termQuery OR termQuery" if all direct children are "StreamFilter"
            filter.filters().stream().map(filter1 -> generateFilterClause(filter1, job, query)).forEach(optQueryBuilder -> optQueryBuilder.ifPresent(orBuilder::should));
            return Optional.of(orBuilder);
        case StreamFilter.NAME:
            // Skipping stream filter, will be extracted elsewhere
            return Optional.empty();
        case QueryStringFilter.NAME:
            return Optional.of(QueryBuilders.queryStringQuery(this.queryStringDecorators.decorate(((QueryStringFilter) filter).query(), job, query)));
    }
    return Optional.empty();
}

Example 2 with Filter

use of org.graylog.plugins.views.search.Filter in project graylog2-server by Graylog2.

the class ElasticsearchBackend method checkForFailedShards.

private Optional<ElasticsearchException> checkForFailedShards(MultiSearchResponse.Item multiSearchResponse) {
    if (multiSearchResponse.isFailure()) {
        return Optional.of(new ElasticsearchException(multiSearchResponse.getFailureMessage(), multiSearchResponse.getFailure()));
    }
    final SearchResponse searchResponse = multiSearchResponse.getResponse();
    if (searchResponse != null && searchResponse.getFailedShards() > 0) {
        final List<Throwable> shardFailures = Arrays.stream(searchResponse.getShardFailures()).map(ShardOperationFailedException::getCause).collect(Collectors.toList());
        final List<String> nonNumericFieldErrors = shardFailures.stream().filter(shardFailure -> shardFailure.getMessage().contains("Expected numeric type on field")).map(Throwable::getMessage).distinct().collect(Collectors.toList());
        if (!nonNumericFieldErrors.isEmpty()) {
            return Optional.of(new FieldTypeException("Unable to perform search query: ", nonNumericFieldErrors));
        }
        final List<String> errors = shardFailures.stream().map(Throwable::getMessage).distinct().collect(Collectors.toList());
        return Optional.of(new ElasticsearchException("Unable to perform search query: ", errors));
    }
    return Optional.empty();
}

Example 3 with Filter

use of org.graylog.plugins.views.search.Filter in project graylog2-server by Graylog2.

the class QueryEntity method shallowMappedFilter.

// TODO: This code assumes that we only use shallow filters for streams.
// If this ever changes, we need to implement a mapper that can handle filter trees.
private Filter shallowMappedFilter(Map<EntityDescriptor, Object> nativeEntities) {
    return Optional.ofNullable(filter()).map(optFilter -> {
        Set<Filter> newFilters = optFilter.filters().stream().map(filter -> {
            if (filter.type().matches(StreamFilter.NAME)) {
                final StreamFilter streamFilter = (StreamFilter) filter;
                final Stream stream = (Stream) nativeEntities.get(EntityDescriptor.create(streamFilter.streamId(), ModelTypes.STREAM_V1));
                if (Objects.isNull(stream)) {
                    throw new ContentPackException("Could not find matching stream id: " + streamFilter.streamId());
                }
                return streamFilter.toBuilder().streamId(stream.getId()).build();
            }
            return filter;
        }).collect(Collectors.toSet());
        return optFilter.toGenericBuilder().filters(newFilters).build();
    }).orElse(null);
}

Example 4 with Filter

use of org.graylog.plugins.views.search.Filter in project graylog2-server by Graylog2.

the class ElasticsearchBackend method generateFilterClause.

// TODO make pluggable
public Optional<QueryBuilder> generateFilterClause(Filter filter, SearchJob job, Query query) {
    if (filter == null) {
        return Optional.empty();
    }
    switch(filter.type()) {
        case AndFilter.NAME:
            final BoolQueryBuilder andBuilder = QueryBuilders.boolQuery();
            filter.filters().stream().map(filter1 -> generateFilterClause(filter1, job, query)).forEach(optQueryBuilder -> optQueryBuilder.ifPresent(andBuilder::must));
            return Optional.of(andBuilder);
        case OrFilter.NAME:
            final BoolQueryBuilder orBuilder = QueryBuilders.boolQuery();
            // TODO for the common case "any of these streams" we can optimize the filter into
            // a single "termsQuery" instead of "termQuery OR termQuery" if all direct children are "StreamFilter"
            filter.filters().stream().map(filter1 -> generateFilterClause(filter1, job, query)).forEach(optQueryBuilder -> optQueryBuilder.ifPresent(orBuilder::should));
            return Optional.of(orBuilder);
        case StreamFilter.NAME:
            // Skipping stream filter, will be extracted elsewhere
            return Optional.empty();
        case QueryStringFilter.NAME:
            return Optional.of(QueryBuilders.queryStringQuery(this.queryStringDecorators.decorate(((QueryStringFilter) filter).query(), job, query)));
    }
    return Optional.empty();
}

Example 5 with Filter

use of org.graylog.plugins.views.search.Filter in project graylog2-server by Graylog2.

the class ElasticsearchBackend method doRun.

@Override
public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {
    if (query.searchTypes().isEmpty()) {
        return QueryResult.builder().query(query).searchTypes(Collections.emptyMap()).errors(new HashSet<>(queryContext.errors())).build();
    }
    LOG.debug("Running query {} for job {}", query.id(), job.getId());
    final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
    final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
    final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
    final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());
    final List<Search> searches = searchTypeIds.stream().map(searchTypeId -> {
        final Set<String> affectedIndicesForSearchType = query.searchTypes().stream().filter(s -> s.id().equalsIgnoreCase(searchTypeId)).findFirst().flatMap(searchType -> {
            if (searchType.effectiveStreams().isEmpty() && !query.globalOverride().flatMap(GlobalOverride::timerange).isPresent() && !searchType.timerange().isPresent()) {
                return Optional.empty();
            }
            final Set<String> usedStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
            return Optional.of(indexLookup.indexNamesForStreamsInTimeRange(usedStreamIds, query.effectiveTimeRange(searchType)));
        }).orElse(affectedIndices);
        return new Search.Builder(searchTypeQueries.get(searchTypeId).toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(affectedIndicesForSearchType.isEmpty() ? Collections.singleton("") : affectedIndicesForSearchType).allowNoIndices(false).ignoreUnavailable(false).build();
    }).collect(Collectors.toList());
    final MultiSearch.Builder multiSearchBuilder = new MultiSearch.Builder(searches);
    final MultiSearchResult result = JestUtils.execute(jestClient, multiSearchBuilder.build(), () -> "Unable to perform search query: ");
    for (SearchType searchType : query.searchTypes()) {
        final String searchTypeId = searchType.id();
        final Provider<ESSearchTypeHandler<? extends SearchType>> handlerProvider = elasticsearchSearchTypeHandlers.get(searchType.type());
        if (handlerProvider == null) {
            LOG.error("Unknown search type '{}', cannot convert query result.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        if (isSearchTypeWithError(queryContext, searchTypeId)) {
            LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        // we create a new instance because some search type handlers might need to track information between generating the query and
        // processing its result, such as aggregations, which depend on the name and type
        final ESSearchTypeHandler<? extends SearchType> handler = handlerProvider.get();
        final int searchTypeIndex = searchTypeIds.indexOf(searchTypeId);
        final MultiSearchResult.MultiSearchResponse multiSearchResponse = result.getResponses().get(searchTypeIndex);
        if (multiSearchResponse.isError) {
            ElasticsearchException e = JestUtils.specificException(() -> "Search type returned error: ", multiSearchResponse.error);
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else if (checkForFailedShards(multiSearchResponse.searchResult).isPresent()) {
            ElasticsearchException e = checkForFailedShards(multiSearchResponse.searchResult).get();
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else {
            final SearchType.Result searchTypeResult = handler.extractResult(job, query, searchType, multiSearchResponse.searchResult, queryContext);
            if (searchTypeResult != null) {
                resultsMap.put(searchTypeId, searchTypeResult);
            }
        }
    }
    LOG.debug("Query {} ran for job {}", query.id(), job.getId());
    return QueryResult.builder().query(query).searchTypes(resultsMap).errors(new HashSet<>(queryContext.errors())).build();
}