Examples with SearchHits org.elasticsearch.search.SearchHits used on opensource projects

Example 36 with SearchHits

use of org.elasticsearch.search.SearchHits in project herd by FINRAOS.

the class IndexSearchDaoTest method testIndexSearch.

private void testIndexSearch(Set<String> fields, Set<String> match, List<IndexSearchFilter> searchFilters, List<String> facetList, boolean isHitHighlightingEnabled, boolean testExceptions, boolean setInvalidSearchResultIndexName, boolean isColumnFields) throws IOException {
    // Build the mocks
    SearchRequestBuilder searchRequestBuilder = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSource = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSize = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSorting = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithHighlighting = mock(SearchRequestBuilder.class);
    SearchResponse searchResponse = mock(SearchResponse.class);
    SearchHits searchHits = mock(SearchHits.class);
    SearchHit searchHit1 = mock(SearchHit.class);
    SearchHit searchHit2 = mock(SearchHit.class);
    SearchShardTarget searchShardTarget1 = mock(SearchShardTarget.class);
    SearchShardTarget searchShardTarget2 = mock(SearchShardTarget.class);
    SearchHit[] searchHitArray = new SearchHit[2];
    searchHitArray[0] = searchHit1;
    searchHitArray[1] = searchHit2;
    HighlightField highlightField = mock(HighlightField.class);
    when(highlightField.getName()).thenReturn("displayName");
    Text[] value = { new Text("match <hlt>fragment</hlt class=\"highlight\">"), new Text("<hlt class=\"highlight\">match</hlt>") };
    when(highlightField.getFragments()).thenReturn(value);
    @SuppressWarnings("unchecked") ListenableActionFuture<SearchResponse> listenableActionFuture = mock(ListenableActionFuture.class);
    final String highlightFieldsConfigValue = "{\"highlightFields\":[{\"fieldName\":\"displayName\",\"fragmentSize\":100,\"matchedFields\":[\"displayName\",\"displayName.stemmed\",\"displayName.ngrams\"],\"numOfFragments\":5}]}";
    final String highlightFieldsColumnMatchConfigValue = "{\"highlightFields\":[{\"fieldName\":\"columnName\",\"fragmentSize\":100,\"matchedFields\":[\"columnName\",\"columnName.stemmed\",\"columnName.ngrams\"],\"numOfFragments\":5}]}";
    // Mock the call to external methods
    when(configurationHelper.getProperty(ConfigurationValue.TAG_SHORT_DESCRIPTION_LENGTH, Integer.class)).thenReturn(300);
    when(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_DEFINITION_SHORT_DESCRIPTION_LENGTH, Integer.class)).thenReturn(300);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_NGRAMS)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_STEMMED)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_SHINGLES)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_PRETAGS)).thenReturn("<hlt class=\"highlight\">");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_POSTTAGS)).thenReturn("</hlt>");
    if (match != null && match.contains(MATCH_COLUMN)) {
        when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_COLUMN_MATCH_HIGHLIGHT_FIELDS)).thenReturn(highlightFieldsColumnMatchConfigValue);
    } else {
        when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_FIELDS)).thenReturn(highlightFieldsConfigValue);
    }
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_SLOP, Integer.class)).thenReturn(5);
    Map<String, String> fieldsBoostMap = new HashMap<>();
    fieldsBoostMap.put("displayName", "1.0");
    if (isColumnFields) {
        fieldsBoostMap.put(COLUMNS_NAME_FIELD, "1.0");
        fieldsBoostMap.put(SCHEMA_COLUMNS_NAME_FIELD, "1.0");
    }
    if (testExceptions) {
        when(jsonHelper.unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}")).thenThrow(new IOException());
    } else {
        when(jsonHelper.unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}")).thenReturn(fieldsBoostMap);
    }
    IndexSearchHighlightField indexSearchHighlightField = new IndexSearchHighlightField("displayName", 100, Arrays.asList("displayName", "displayName.stemmed", "displayName.ngrams"), 5);
    IndexSearchHighlightFields highlightFields = new IndexSearchHighlightFields(Collections.singletonList(indexSearchHighlightField));
    IndexSearchHighlightField indexSearchHighlightFieldColumnOnly = new IndexSearchHighlightField("columnName", 100, Arrays.asList("columnName", "columnName.stemmed", "columnName.ngrams"), 5);
    IndexSearchHighlightFields highlightFieldsColumnOnly = new IndexSearchHighlightFields(Collections.singletonList(indexSearchHighlightFieldColumnOnly));
    if (testExceptions) {
        when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsConfigValue)).thenThrow(new IOException());
    } else {
        if (match != null && match.contains(MATCH_COLUMN)) {
            when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsColumnMatchConfigValue)).thenReturn(highlightFieldsColumnOnly);
        } else {
            when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsConfigValue)).thenReturn(highlightFields);
        }
    }
    when(searchRequestBuilder.setSource(any(SearchSourceBuilder.class))).thenReturn(searchRequestBuilderWithSource);
    when(searchRequestBuilderWithSource.setSize(SEARCH_RESULT_SIZE)).thenReturn(searchRequestBuilderWithSize);
    when(searchRequestBuilderWithSize.addSort(any(SortBuilder.class))).thenReturn(searchRequestBuilderWithSorting);
    when(searchRequestBuilderWithSorting.highlighter(any(HighlightBuilder.class))).thenReturn(searchRequestBuilderWithHighlighting);
    when(searchRequestBuilder.execute()).thenReturn(listenableActionFuture);
    when(listenableActionFuture.actionGet()).thenReturn(searchResponse);
    when(searchResponse.getHits()).thenReturn(searchHits);
    when(searchHits.hits()).thenReturn(searchHitArray);
    Map<String, Object> sourceMap1 = new HashMap<>();
    Map<String, Object> tagTypeMap = new HashMap<>();
    tagTypeMap.put(CODE, TAG_TYPE_CODE);
    sourceMap1.put(TAG_TYPE, tagTypeMap);
    when(searchHit1.sourceAsMap()).thenReturn(sourceMap1);
    Map<String, Object> sourceMap2 = new HashMap<>();
    Map<String, Object> businessObjectDefinitionMap = new HashMap<>();
    businessObjectDefinitionMap.put(CODE, NAMESPACE_CODE);
    sourceMap2.put(NAMESPACE, businessObjectDefinitionMap);
    when(searchHit2.sourceAsMap()).thenReturn(sourceMap2);
    when(searchHit1.getShard()).thenReturn(searchShardTarget1);
    when(searchHit2.getShard()).thenReturn(searchShardTarget2);
    when(searchShardTarget1.getIndex()).thenReturn(TAG_SEARCH_INDEX_NAME);
    when(searchShardTarget2.getIndex()).thenReturn(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME);
    when(searchHits.getTotalHits()).thenReturn(200L);
    Map<String, HighlightField> highlightFieldMap = new HashMap<>();
    highlightFieldMap.put("displayName", highlightField);
    when(searchHit1.getHighlightFields()).thenReturn(highlightFieldMap);
    when(searchHit2.getHighlightFields()).thenReturn(highlightFieldMap);
    // Create index search request
    final IndexSearchRequest indexSearchRequest = new IndexSearchRequest(SEARCH_TERM, searchFilters, facetList, isHitHighlightingEnabled);
    List<TagTypeIndexSearchResponseDto> tagTypeIndexSearchResponseDtos = Collections.singletonList(new TagTypeIndexSearchResponseDto("code", Collections.singletonList(new TagIndexSearchResponseDto("tag1", 1, null)), null));
    List<ResultTypeIndexSearchResponseDto> resultTypeIndexSearchResponseDto = Collections.singletonList(new ResultTypeIndexSearchResponseDto("type", 1, null));
    when(elasticsearchHelper.getNestedTagTagIndexSearchResponseDto(searchResponse)).thenReturn(tagTypeIndexSearchResponseDtos);
    when(elasticsearchHelper.getResultTypeIndexSearchResponseDto(searchResponse)).thenReturn(resultTypeIndexSearchResponseDto);
    when(elasticsearchHelper.getFacetsResponse(any(ElasticsearchResponseDto.class), eq(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME), eq(TAG_SEARCH_INDEX_NAME))).thenCallRealMethod();
    when(elasticsearchHelper.addIndexSearchFilterBooleanClause(any(), any(), any())).thenCallRealMethod();
    when(elasticsearchHelper.addFacetFieldAggregations(any(), any(SearchRequestBuilder.class))).thenReturn(searchRequestBuilder);
    when(herdSearchQueryHelper.determineNegationTermsPresent(any(IndexSearchRequest.class))).thenCallRealMethod();
    when(herdSearchQueryHelper.extractNegationTerms(any(IndexSearchRequest.class))).thenCallRealMethod();
    when(herdSearchQueryHelper.extractSearchPhrase(any(IndexSearchRequest.class))).thenCallRealMethod();
    SearchResult searchResult = mock(SearchResult.class);
    when(jestClientHelper.searchExecute(any())).thenReturn(searchResult);
    List<SearchResult.Hit<Map, Void>> searchHitList = new ArrayList<>();
    Gson gson = new Gson();
    Map<String, Object> map = new HashMap<>();
    map.put(DISPLAY_NAME_SOURCE, "Display Name");
    Map<String, Object> codeMap = new HashMap<>();
    codeMap.put(CODE, "Code");
    map.put(TAG_TYPE, codeMap);
    map.put(NAMESPACE, codeMap);
    JsonElement element = gson.toJsonTree(map);
    List<String> highlightList = new ArrayList<>();
    highlightList.add("Highlight 1");
    Map<String, List<String>> highlightMap = new HashMap<>();
    highlightMap.put("field", highlightList);
    SearchResult.Hit<Map, Void> hit1 = new SearchResult(gson).new Hit(HashMap.class, element, HashMap.class, null, highlightMap, null, setInvalidSearchResultIndexName ? INVALID_VALUE : TAG_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME, "type", 1.0);
    SearchResult.Hit<Map, Void> hit2 = new SearchResult(gson).new Hit(HashMap.class, element, HashMap.class, null, highlightMap, null, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, "type", 2.0);
    searchHitList.add(hit1);
    searchHitList.add(hit2);
    when(searchResult.getHits(Map.class)).thenReturn(searchHitList);
    when(searchResult.getTotal()).thenReturn(200L);
    MetricAggregation metricAggregation = mock(MetricAggregation.class);
    TermsAggregation termsAggregation = mock(TermsAggregation.class);
    when(searchResult.getAggregations()).thenReturn(metricAggregation);
    when(metricAggregation.getTermsAggregation(TAG_TYPE_FACET_AGGS)).thenReturn(termsAggregation);
    List<TermsAggregation.Entry> buckets = new ArrayList<>();
    TermsAggregation.Entry entry1 = mock(TermsAggregation.Entry.class);
    TermsAggregation.Entry entry2 = mock(TermsAggregation.Entry.class);
    buckets.add(entry1);
    buckets.add(entry2);
    when(termsAggregation.getBuckets()).thenReturn(buckets);
    // Call the method under test
    IndexSearchResponse indexSearchResponse = indexSearchDao.indexSearch(indexSearchRequest, fields, match, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME);
    List<IndexSearchResult> indexSearchResults = indexSearchResponse.getIndexSearchResults();
    assertThat("Index search results list is null.", indexSearchResults, not(nullValue()));
    assertThat(indexSearchResponse.getTotalIndexSearchResults(), is(200L));
    // Verify external method calls.
    verify(herdSearchQueryHelper).determineNegationTermsPresent(indexSearchRequest);
    verify(herdSearchQueryHelper).extractNegationTerms(indexSearchRequest);
    verify(herdSearchQueryHelper).extractSearchPhrase(indexSearchRequest);
    verify(configurationHelper).getProperty(ConfigurationValue.TAG_SHORT_DESCRIPTION_LENGTH, Integer.class);
    verify(configurationHelper).getProperty(ConfigurationValue.BUSINESS_OBJECT_DEFINITION_SHORT_DESCRIPTION_LENGTH, Integer.class);
    verify(configurationHelper, times(3)).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_STEMMED);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_NGRAMS);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_SHINGLES);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class);
    verify(configurationHelper, times(2)).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class);
    verify(configurationHelper, times(5)).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_SLOP, Integer.class);
    verify(jsonHelper, times(5)).unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}");
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getIndexSearchFilters())) {
        verify(elasticsearchHelper).addIndexSearchFilterBooleanClause(indexSearchRequest.getIndexSearchFilters(), BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME);
    }
    if (indexSearchRequest.isEnableHitHighlighting() != null) {
        verifyHitHighlightingInteractions(searchRequestBuilder, indexSearchRequest.isEnableHitHighlighting(), match);
    }
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        verify(elasticsearchHelper).addFacetFieldAggregations(any(), any(SearchRequestBuilder.class));
        if (indexSearchRequest.getFacetFields().contains(ElasticsearchHelper.TAG_FACET)) {
            verify(elasticsearchHelper).getNestedTagTagIndexSearchResponseDto(searchResult);
            verify(elasticsearchHelper).getTagTagIndexSearchResponseDto(searchResult);
        }
        if (indexSearchRequest.getFacetFields().contains(ElasticsearchHelper.RESULT_TYPE_FACET)) {
            verify(elasticsearchHelper).getResultTypeIndexSearchResponseDto(searchResult);
        }
        verify(elasticsearchHelper).getFacetsResponse(any(ElasticsearchResponseDto.class), eq(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME), eq(TAG_SEARCH_INDEX_NAME));
    }
    verify(jestClientHelper).searchExecute(any());
    verify(searchResult).getTotal();
    verify(searchResult).getHits(Map.class);
    verifyNoMoreInteractions(searchRequestBuilder, searchRequestBuilderWithSource, searchRequestBuilderWithSize, searchRequestBuilderWithSorting, searchRequestBuilderWithHighlighting, searchResponse, searchHits, searchHit1, searchHit2, searchShardTarget1, searchShardTarget2, highlightField, listenableActionFuture, searchResult, metricAggregation, termsAggregation, entry1, entry2);
    verifyNoMoreInteractionsHelper();
}

Example 37 with SearchHits

use of org.elasticsearch.search.SearchHits in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method fetchAllEntities.

private <T> List<T> fetchAllEntities(final String docType, final BuildEntity<T> be, final FilterBuilder filter) throws CalFacadeException {
    final SearchRequestBuilder srb = getClient().prepareSearch(targetIndex);
    srb.setTypes(docType);
    if (debug) {
        debug("fetchAllEntities: srb=" + srb);
    }
    int tries = 0;
    // int ourPos = 0;
    final int ourCount = maxFetchCount;
    final List<T> res = new ArrayList<>();
    SearchResponse scrollResp = srb.setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setPostFilter(filter).setSize(ourCount).execute().actionGet();
    if (scrollResp.status() != RestStatus.OK) {
        if (debug) {
            debug("Search returned status " + scrollResp.status());
        }
    }
    for (; ; ) {
        if (tries > absoluteMaxTries) {
            // huge count or we screwed up
            warn("Indexer: too many tries");
            break;
        }
        scrollResp = getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
        if (scrollResp.status() != RestStatus.OK) {
            if (debug) {
                debug("Search returned status " + scrollResp.status());
            }
        }
        final SearchHits hits = scrollResp.getHits();
        // Break condition: No hits are returned
        if (hits.hits().length == 0) {
            break;
        }
        for (final SearchHit hit : hits) {
            // Handle the hit...
            final T ent = be.make(getEntityBuilder(hit.sourceAsMap()));
            if (ent == null) {
                // No access
                continue;
            }
            res.add(ent);
        // ourPos++;
        }
        tries++;
    }
    return res;
}

Example 38 with SearchHits

use of org.elasticsearch.search.SearchHits in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method multiColFetch.

/* ========================================================================
   *                   private methods
   * ======================================================================== */
private SearchHits multiColFetch(final List<String> hrefs) throws CalFacadeException {
    final int batchSize = hrefs.size();
    final SearchRequestBuilder srb = getClient().prepareSearch(searchIndexes);
    final TermsQueryBuilder tqb = new TermsQueryBuilder(ESQueryFilter.getJname(PropertyInfoIndex.HREF), hrefs);
    srb.setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(tqb);
    srb.setFrom(0);
    srb.setSize(batchSize);
    if (debug) {
        debug("MultiColFetch: targetIndex=" + targetIndex + "; srb=" + srb);
    }
    final SearchResponse resp = srb.execute().actionGet();
    if (resp.status() != RestStatus.OK) {
        if (debug) {
            debug("Search returned status " + resp.status());
        }
        return null;
    }
    final SearchHits hits = resp.getHits();
    if ((hits.getHits() == null) || (hits.getHits().length == 0)) {
        return null;
    }
    // Break condition: No hits are returned
    if (hits.hits().length == 0) {
        return null;
    }
    return hits;
}

Example 39 with SearchHits

use of org.elasticsearch.search.SearchHits in project nuxeo-drive-server by nuxeo.

the class ESAuditChangeFinder method getUpperBound.

@Override
public long getUpperBound() {
    SearchRequest request = new SearchRequest(getESIndexName()).types(ElasticSearchConstants.ENTRY_TYPE).searchType(SearchType.DFS_QUERY_THEN_FETCH);
    // TODO refactor this to use max clause
    request.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).sort("id", SortOrder.DESC).size(1));
    logSearchRequest(request);
    SearchResponse searchResponse = getClient().search(request);
    logSearchResponse(searchResponse);
    List<LogEntry> entries = new ArrayList<>();
    SearchHits hits = searchResponse.getHits();
    ObjectMapper mapper = new ObjectMapper();
    for (SearchHit hit : hits) {
        try {
            entries.add(mapper.readValue(hit.getSourceAsString(), LogEntryImpl.class));
        } catch (IOException e) {
            log.error("Error while reading Audit Entry from ES", e);
        }
    }
    return entries.size() > 0 ? entries.get(0).getId() : -1;
}

Example 40 with SearchHits

use of org.elasticsearch.search.SearchHits in project muikku by otavanopisto.

the class ElasticSearchProvider method searchWorkspaces.

@Override
public SearchResult searchWorkspaces(String schoolDataSource, String subjectIdentifier, int courseNumber) {
    BoolQueryBuilder query = boolQuery();
    query.must(termQuery("published", Boolean.TRUE));
    query.must(termQuery("subjectIdentifier", subjectIdentifier));
    query.must(termQuery("courseNumber", courseNumber));
    // query.must(termQuery("access", WorkspaceAccess.LOGGED_IN));
    SearchRequestBuilder requestBuilder = elasticClient.prepareSearch("muikku").setTypes("Workspace").setFrom(0).setSize(50).setQuery(query);
    // logger.log(Level.INFO, "searchWorkspaces query: " + requestBuilder.internalBuilder());
    SearchResponse response = requestBuilder.execute().actionGet();
    List<Map<String, Object>> searchResults = new ArrayList<Map<String, Object>>();
    SearchHits searchHits = response.getHits();
    SearchHit[] results = searchHits.getHits();
    long totalHits = searchHits.getTotalHits();
    for (SearchHit hit : results) {
        Map<String, Object> hitSource = hit.getSource();
        hitSource.put("indexType", hit.getType());
        searchResults.add(hitSource);
    }
    SearchResult result = new SearchResult(0, 50, searchResults, totalHits);
    return result;
}