Examples with SearchType org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchType used on opensource projects

Example 6 with SearchType

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchType in project elasticsearch by elastic.

the class SearchScrollIT method testDeepScrollingDoesNotBlowUp.

/**
     * Tests that we use an optimization shrinking the batch to the size of the shard. Thus the Integer.MAX_VALUE window doesn't OOM us.
     */
public void testDeepScrollingDoesNotBlowUp() throws Exception {
    client().prepareIndex("index", "type", "1").setSource("field", "value").setRefreshPolicy(IMMEDIATE).execute().get();
    /*
         * Disable the max result window setting for this test because it'll reject the search's unreasonable batch size. We want
         * unreasonable batch sizes to just OOM.
         */
    client().admin().indices().prepareUpdateSettings("index").setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)).get();
    for (SearchType searchType : SearchType.values()) {
        SearchRequestBuilder builder = client().prepareSearch("index").setSearchType(searchType).setQuery(QueryBuilders.matchAllQuery()).setSize(Integer.MAX_VALUE).setScroll("1m");
        SearchResponse response = builder.execute().actionGet();
        try {
            ElasticsearchAssertions.assertHitCount(response, 1L);
        } finally {
            String scrollId = response.getScrollId();
            if (scrollId != null) {
                clearScroll(scrollId);
            }
        }
    }
}

Example 7 with SearchType

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchType in project elasticsearch by elastic.

the class ChildQuerySearchIT method testSimpleQueryRewrite.

public void testSimpleQueryRewrite() throws Exception {
    assertAcked(prepareCreate("test").addMapping("parent", "p_field", "type=keyword").addMapping("child", "_parent", "type=parent", "c_field", "type=keyword"));
    ensureGreen();
    // index simple data
    int childId = 0;
    for (int i = 0; i < 10; i++) {
        String parentId = String.format(Locale.ROOT, "p%03d", i);
        client().prepareIndex("test", "parent", parentId).setSource("p_field", parentId).get();
        int j = childId;
        for (; j < childId + 50; j++) {
            String childUid = String.format(Locale.ROOT, "c%03d", j);
            client().prepareIndex("test", "child", childUid).setSource("c_field", childUid).setParent(parentId).get();
        }
        childId = j;
    }
    refresh();
    SearchType[] searchTypes = new SearchType[] { SearchType.QUERY_THEN_FETCH, SearchType.DFS_QUERY_THEN_FETCH };
    for (SearchType searchType : searchTypes) {
        SearchResponse searchResponse = client().prepareSearch("test").setSearchType(searchType).setQuery(hasChildQuery("child", prefixQuery("c_field", "c"), ScoreMode.Max)).addSort("p_field", SortOrder.ASC).setSize(5).get();
        assertNoFailures(searchResponse);
        assertThat(searchResponse.getHits().getTotalHits(), equalTo(10L));
        assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("p000"));
        assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("p001"));
        assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("p002"));
        assertThat(searchResponse.getHits().getHits()[3].getId(), equalTo("p003"));
        assertThat(searchResponse.getHits().getHits()[4].getId(), equalTo("p004"));
        searchResponse = client().prepareSearch("test").setSearchType(searchType).setQuery(hasParentQuery("parent", prefixQuery("p_field", "p"), true)).addSort("c_field", SortOrder.ASC).setSize(5).get();
        assertNoFailures(searchResponse);
        assertThat(searchResponse.getHits().getTotalHits(), equalTo(500L));
        assertThat(searchResponse.getHits().getHits()[0].getId(), equalTo("c000"));
        assertThat(searchResponse.getHits().getHits()[1].getId(), equalTo("c001"));
        assertThat(searchResponse.getHits().getHits()[2].getId(), equalTo("c002"));
        assertThat(searchResponse.getHits().getHits()[3].getId(), equalTo("c003"));
        assertThat(searchResponse.getHits().getHits()[4].getId(), equalTo("c004"));
    }
}

Example 8 with SearchType

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchType in project graylog2-server by Graylog2.

the class ESMessageList method doExtractResult.

@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, MessageList searchType, org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchResponse result, Aggregations aggregations, ESGeneratedQueryContext queryContext) {
    final List<ResultMessageSummary> messages = StreamSupport.stream(result.getHits().spliterator(), false).map(ESMessageList::resultMessageFromSearchHit).map((resultMessage) -> ResultMessageSummary.create(resultMessage.highlightRanges, resultMessage.getMessage().getFields(), resultMessage.getIndex())).collect(Collectors.toList());
    final String undecoratedQueryString = query.query().queryString();
    final String queryString = this.esQueryDecorators.decorate(undecoratedQueryString, job, query);
    final DateTime from = query.effectiveTimeRange(searchType).getFrom();
    final DateTime to = query.effectiveTimeRange(searchType).getTo();
    final SearchResponse searchResponse = SearchResponse.create(undecoratedQueryString, queryString, Collections.emptySet(), messages, Collections.emptySet(), 0, result.getHits().getTotalHits().value, from, to);
    final SearchResponse decoratedSearchResponse = decoratorProcessor.decorateSearchResponse(searchResponse, searchType.decorators());
    final MessageList.Result.Builder resultBuilder = MessageList.Result.result(searchType.id()).messages(decoratedSearchResponse.messages()).effectiveTimerange(AbsoluteRange.create(from, to)).totalResults(decoratedSearchResponse.totalResults());
    return searchType.name().map(resultBuilder::name).orElse(resultBuilder).build();
}

Example 9 with SearchType

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchType in project graylog2-server by Graylog2.

the class IndicesAdapterES7 method indexRangeStatsOfIndex.

@Override
public IndexRangeStats indexRangeStatsOfIndex(String index) {
    final FilterAggregationBuilder builder = AggregationBuilders.filter("agg", QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.min("ts_min").field(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.max("ts_max").field(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.terms("streams").size(Integer.MAX_VALUE).field(Message.FIELD_STREAMS));
    final SearchSourceBuilder query = SearchSourceBuilder.searchSource().aggregation(builder).size(0);
    final SearchRequest request = new SearchRequest().source(query).indices(index).searchType(SearchType.DFS_QUERY_THEN_FETCH).indicesOptions(IndicesOptions.lenientExpandOpen());
    final SearchResponse result = client.execute((c, requestOptions) -> c.search(request, requestOptions), "Couldn't build index range of index " + index);
    if (result.getTotalShards() == 0 || result.getAggregations() == null) {
        throw new IndexNotFoundException("Couldn't build index range of index " + index + " because it doesn't exist.");
    }
    final Filter f = result.getAggregations().get("agg");
    if (f == null) {
        throw new IndexNotFoundException("Couldn't build index range of index " + index + " because it doesn't exist.");
    } else if (f.getDocCount() == 0L) {
        LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index);
        return IndexRangeStats.EMPTY;
    }
    final Min minAgg = f.getAggregations().get("ts_min");
    final long minUnixTime = new Double(minAgg.getValue()).longValue();
    final DateTime min = new DateTime(minUnixTime, DateTimeZone.UTC);
    final Max maxAgg = f.getAggregations().get("ts_max");
    final long maxUnixTime = new Double(maxAgg.getValue()).longValue();
    final DateTime max = new DateTime(maxUnixTime, DateTimeZone.UTC);
    // make sure we return an empty list, so we can differentiate between old indices that don't have this information
    // and newer ones that simply have no streams.
    final Terms streams = f.getAggregations().get("streams");
    final List<String> streamIds = streams.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKeyAsString).collect(toList());
    return IndexRangeStats.create(min, max, streamIds);
}

Example 10 with SearchType

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchType in project graylog2-server by Graylog2.

the class ElasticsearchBackend method generate.

@Override
public ESGeneratedQueryContext generate(SearchJob job, Query query, SearchConfig searchConfig) {
    final BackendQuery backendQuery = query.query();
    validateQueryTimeRange(query, searchConfig);
    final Set<SearchType> searchTypes = query.searchTypes();
    final String queryString = this.queryStringDecorators.decorate(backendQuery.queryString(), job, query);
    final QueryBuilder normalizedRootQuery = normalizeQueryString(queryString);
    final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().filter(normalizedRootQuery);
    // add the optional root query filters
    generateFilterClause(query.filter(), job, query).map(boolQuery::filter);
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(boolQuery).from(0).size(0).trackTotalHits(true);
    final ESGeneratedQueryContext queryContext = queryContextFactory.create(this, searchSourceBuilder, job, query);
    for (SearchType searchType : searchTypes) {
        final Optional<SearchTypeError> searchTypeError = validateSearchType(query, searchType, searchConfig);
        if (searchTypeError.isPresent()) {
            LOG.error("Invalid search type {} for elasticsearch backend, cannot generate query part. Skipping this search type.", searchType.type());
            queryContext.addError(searchTypeError.get());
            continue;
        }
        final SearchSourceBuilder searchTypeSourceBuilder = queryContext.searchSourceBuilder(searchType);
        final Set<String> effectiveStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
        final BoolQueryBuilder searchTypeOverrides = QueryBuilders.boolQuery().must(searchTypeSourceBuilder.query()).must(Objects.requireNonNull(TimeRangeQueryFactory.create(query.effectiveTimeRange(searchType)), "Timerange for search type " + searchType.id() + " cannot be found in query or search type.")).must(QueryBuilders.termsQuery(Message.FIELD_STREAMS, effectiveStreamIds));
        searchType.query().ifPresent(searchTypeQuery -> {
            final String searchTypeQueryString = this.queryStringDecorators.decorate(searchTypeQuery.queryString(), job, query);
            final QueryBuilder normalizedSearchTypeQuery = normalizeQueryString(searchTypeQueryString);
            searchTypeOverrides.must(normalizedSearchTypeQuery);
        });
        searchTypeSourceBuilder.query(searchTypeOverrides);
        final String type = searchType.type();
        final Provider<ESSearchTypeHandler<? extends SearchType>> searchTypeHandler = elasticsearchSearchTypeHandlers.get(type);
        if (searchTypeHandler == null) {
            LOG.error("Unknown search type {} for elasticsearch backend, cannot generate query part. Skipping this search type.", type);
            queryContext.addError(new SearchTypeError(query, searchType.id(), "Unknown search type '" + type + "' for elasticsearch backend, cannot generate query"));
            continue;
        }
        if (isSearchTypeWithError(queryContext, searchType.id())) {
            LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        searchTypeHandler.get().generateQueryPart(job, query, searchType, queryContext);
    }
    return queryContext;
}