Examples with MultiSearchResponse org.elasticsearch.action.search.MultiSearchResponse used on opensource projects

Example 16 with MultiSearchResponse

use of org.elasticsearch.action.search.MultiSearchResponse in project graylog2-server by Graylog2.

the class ElasticsearchBackend method doRun.

@Override
public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {
    if (query.searchTypes().isEmpty()) {
        return QueryResult.builder().query(query).searchTypes(Collections.emptyMap()).errors(new HashSet<>(queryContext.errors())).build();
    }
    LOG.debug("Running query {} for job {}", query.id(), job.getId());
    final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
    final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
    final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
    final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());
    final List<SearchRequest> searches = searchTypeIds.stream().map(searchTypeId -> {
        final Set<String> affectedIndicesForSearchType = query.searchTypes().stream().filter(s -> s.id().equalsIgnoreCase(searchTypeId)).findFirst().flatMap(searchType -> {
            if (searchType.effectiveStreams().isEmpty() && !query.globalOverride().flatMap(GlobalOverride::timerange).isPresent() && !searchType.timerange().isPresent()) {
                return Optional.empty();
            }
            final Set<String> usedStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
            return Optional.of(indexLookup.indexNamesForStreamsInTimeRange(usedStreamIds, query.effectiveTimeRange(searchType)));
        }).orElse(affectedIndices);
        Set<String> indices = affectedIndicesForSearchType.isEmpty() ? Collections.singleton("") : affectedIndicesForSearchType;
        return new SearchRequest().source(searchTypeQueries.get(searchTypeId)).indices(indices.toArray(new String[0])).indicesOptions(IndicesOptions.fromOptions(false, false, true, false));
    }).collect(Collectors.toList());
    final List<MultiSearchResponse.Item> results = client.msearch(searches, "Unable to perform search query: ");
    for (SearchType searchType : query.searchTypes()) {
        final String searchTypeId = searchType.id();
        final Provider<ESSearchTypeHandler<? extends SearchType>> handlerProvider = elasticsearchSearchTypeHandlers.get(searchType.type());
        if (handlerProvider == null) {
            LOG.error("Unknown search type '{}', cannot convert query result.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        if (isSearchTypeWithError(queryContext, searchTypeId)) {
            LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        // we create a new instance because some search type handlers might need to track information between generating the query and
        // processing its result, such as aggregations, which depend on the name and type
        final ESSearchTypeHandler<? extends SearchType> handler = handlerProvider.get();
        final int searchTypeIndex = searchTypeIds.indexOf(searchTypeId);
        final MultiSearchResponse.Item multiSearchResponse = results.get(searchTypeIndex);
        if (multiSearchResponse.isFailure()) {
            ElasticsearchException e = new ElasticsearchException("Search type returned error: ", multiSearchResponse.getFailure());
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else if (checkForFailedShards(multiSearchResponse).isPresent()) {
            ElasticsearchException e = checkForFailedShards(multiSearchResponse).get();
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else {
            final SearchType.Result searchTypeResult = handler.extractResult(job, query, searchType, multiSearchResponse.getResponse(), queryContext);
            if (searchTypeResult != null) {
                resultsMap.put(searchTypeId, searchTypeResult);
            }
        }
    }
    LOG.debug("Query {} ran for job {}", query.id(), job.getId());
    return QueryResult.builder().query(query).searchTypes(resultsMap).errors(new HashSet<>(queryContext.errors())).build();
}

Example 17 with MultiSearchResponse

use of org.elasticsearch.action.search.MultiSearchResponse in project graylog2-server by Graylog2.

the class ElasticsearchBackendUsingCorrectIndicesTest method setupSUT.

@Before
public void setupSUT() throws Exception {
    final MultiSearchResponse response = TestMultisearchResponse.fromFixture("successfulResponseWithSingleQuery.json");
    final List<MultiSearchResponse.Item> items = Arrays.stream(response.getResponses()).collect(Collectors.toList());
    when(client.msearch(any(), any())).thenReturn(items);
    final FieldTypesLookup fieldTypesLookup = mock(FieldTypesLookup.class);
    this.backend = new ElasticsearchBackend(handlers, client, indexLookup, new QueryStringDecorators.Fake(), (elasticsearchBackend, ssb, job, query) -> new ESGeneratedQueryContext(elasticsearchBackend, ssb, job, query, fieldTypesLookup), false);
}

Example 18 with MultiSearchResponse

use of org.elasticsearch.action.search.MultiSearchResponse in project graylog2-server by Graylog2.

the class ElasticsearchBackendErrorHandlingTest method deduplicateNumericShardErrorsOnSearchTypeLevel.

@Test
public void deduplicateNumericShardErrorsOnSearchTypeLevel() throws IOException {
    final MultiSearchResponse multiSearchResult = TestMultisearchResponse.fromFixture("errorhandling/numericFailureOnSearchTypeLevel.json");
    final List<MultiSearchResponse.Item> items = Arrays.stream(multiSearchResult.getResponses()).collect(Collectors.toList());
    when(client.msearch(any(), any())).thenReturn(items);
    final QueryResult queryResult = this.backend.doRun(searchJob, query, queryContext);
    final Set<SearchError> errors = queryResult.errors();
    assertThat(errors).isNotNull();
    assertThat(errors).hasSize(1);
    assertThat(errors.stream().map(SearchError::description).collect(Collectors.toList())).containsExactly("Unable to perform search query: " + "\n\nElasticsearch exception [type=illegal_argument_exception, reason=Expected numeric type on field [facility], but got [keyword]].");
}

Example 19 with MultiSearchResponse

use of org.elasticsearch.action.search.MultiSearchResponse in project graylog2-server by Graylog2.

the class ElasticsearchBackendMultiSearchTest method oneFailingSearchTypeReturnsPartialResults.

@Test
public void oneFailingSearchTypeReturnsPartialResults() throws Exception {
    final ESGeneratedQueryContext queryContext = this.elasticsearchBackend.generate(searchJob, query, new SearchConfig(Period.ZERO));
    final MultiSearchResponse response = TestMultisearchResponse.fromFixture("partiallySuccessfulMultiSearchResponse.json");
    final List<MultiSearchResponse.Item> items = Arrays.stream(response.getResponses()).collect(Collectors.toList());
    when(client.msearch(any(), any())).thenReturn(items);
    final QueryResult queryResult = this.elasticsearchBackend.doRun(searchJob, query, queryContext);
    assertThat(queryResult.errors()).hasSize(1);
    final SearchTypeError searchTypeError = (SearchTypeError) new ArrayList<>(queryResult.errors()).get(0);
    assertThat(searchTypeError.description()).isEqualTo("Unable to perform search query: \n" + "\n" + "Elasticsearch exception [type=illegal_argument_exception, reason=Expected numeric type on field [field1], but got [keyword]].");
    assertThat(searchTypeError.searchTypeId()).isEqualTo("pivot1");
    assertThat(queryResult.searchTypes()).containsOnlyKeys("pivot2");
    final PivotResult pivot2Result = (PivotResult) queryResult.searchTypes().get("pivot2");
    assertThat(pivot2Result.rows().get(0)).isEqualTo(PivotResult.Row.builder().key(ImmutableList.of()).source("leaf").addValue(PivotResult.Value.create(Collections.singletonList("max(field2)"), 42.0, true, "row-leaf")).build());
}

Example 20 with MultiSearchResponse

use of org.elasticsearch.action.search.MultiSearchResponse in project graylog2-server by Graylog2.

the class ElasticsearchBackendSearchTypeOverridesTest method overridesInSearchTypeAreIncorporatedIntoGeneratedQueries.

@Test
public void overridesInSearchTypeAreIncorporatedIntoGeneratedQueries() throws IOException {
    final ESGeneratedQueryContext queryContext = this.elasticsearchBackend.generate(searchJob, query, new SearchConfig(Period.ZERO));
    final MultiSearchResponse response = TestMultisearchResponse.fromFixture("successfulMultiSearchResponse.json");
    final List<MultiSearchResponse.Item> items = Arrays.stream(response.getResponses()).collect(Collectors.toList());
    when(client.msearch(any(), any())).thenReturn(items);
    final List<SearchRequest> generatedRequest = run(searchJob, query, queryContext, Collections.emptySet());
    final DocumentContext pivot1 = parse(generatedRequest.get(0).source().toString());
    final DocumentContext pivot2 = parse(generatedRequest.get(1).source().toString());
    assertThat(queryStrings(pivot1)).containsExactly("production:true");
    assertThat(timerangeFrom(pivot1)).containsExactly("2019-09-11 10:31:52.819");
    assertThat(timerangeTo(pivot1)).containsExactly("2019-09-11 10:36:52.823");
    assertThat(streams(pivot1)).containsExactly(Collections.singletonList("stream1"));
    assertThat(queryStrings(pivot2)).containsExactly("production:true", "source:babbage");
    assertThat(timerangeFrom(pivot2)).containsExactly("2018-08-23 08:02:00.247");
    assertThat(timerangeTo(pivot2)).containsExactly("2018-08-23 08:07:00.252");
    assertThat(streams(pivot2)).containsExactly(Collections.singletonList("stream1"));
}