Examples with IndexSearchHighlightFields org.finra.herd.model.dto.IndexSearchHighlightFields used on opensource projects

Example 1 with IndexSearchHighlightFields

use of org.finra.herd.model.dto.IndexSearchHighlightFields in project herd by FINRAOS.

the class IndexSearchDaoTest method testIndexSearch.

private void testIndexSearch(Set<String> fields, Set<String> match, List<IndexSearchFilter> searchFilters, List<String> facetList, boolean isHitHighlightingEnabled, boolean testExceptions, boolean setInvalidSearchResultIndexName, boolean isColumnFields) throws IOException {
    // Build the mocks
    SearchRequestBuilder searchRequestBuilder = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSource = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSize = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSorting = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithHighlighting = mock(SearchRequestBuilder.class);
    SearchResponse searchResponse = mock(SearchResponse.class);
    SearchHits searchHits = mock(SearchHits.class);
    SearchHit searchHit1 = mock(SearchHit.class);
    SearchHit searchHit2 = mock(SearchHit.class);
    SearchShardTarget searchShardTarget1 = mock(SearchShardTarget.class);
    SearchShardTarget searchShardTarget2 = mock(SearchShardTarget.class);
    SearchHit[] searchHitArray = new SearchHit[2];
    searchHitArray[0] = searchHit1;
    searchHitArray[1] = searchHit2;
    HighlightField highlightField = mock(HighlightField.class);
    when(highlightField.getName()).thenReturn("displayName");
    Text[] value = { new Text("match <hlt>fragment</hlt class=\"highlight\">"), new Text("<hlt class=\"highlight\">match</hlt>") };
    when(highlightField.getFragments()).thenReturn(value);
    @SuppressWarnings("unchecked") ListenableActionFuture<SearchResponse> listenableActionFuture = mock(ListenableActionFuture.class);
    final String highlightFieldsConfigValue = "{\"highlightFields\":[{\"fieldName\":\"displayName\",\"fragmentSize\":100,\"matchedFields\":[\"displayName\",\"displayName.stemmed\",\"displayName.ngrams\"],\"numOfFragments\":5}]}";
    final String highlightFieldsColumnMatchConfigValue = "{\"highlightFields\":[{\"fieldName\":\"columnName\",\"fragmentSize\":100,\"matchedFields\":[\"columnName\",\"columnName.stemmed\",\"columnName.ngrams\"],\"numOfFragments\":5}]}";
    // Mock the call to external methods
    when(configurationHelper.getProperty(ConfigurationValue.TAG_SHORT_DESCRIPTION_LENGTH, Integer.class)).thenReturn(300);
    when(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_DEFINITION_SHORT_DESCRIPTION_LENGTH, Integer.class)).thenReturn(300);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_NGRAMS)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_STEMMED)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_SHINGLES)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_PRETAGS)).thenReturn("<hlt class=\"highlight\">");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_POSTTAGS)).thenReturn("</hlt>");
    if (match != null && match.contains(MATCH_COLUMN)) {
        when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_COLUMN_MATCH_HIGHLIGHT_FIELDS)).thenReturn(highlightFieldsColumnMatchConfigValue);
    } else {
        when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_FIELDS)).thenReturn(highlightFieldsConfigValue);
    }
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_SLOP, Integer.class)).thenReturn(5);
    Map<String, String> fieldsBoostMap = new HashMap<>();
    fieldsBoostMap.put("displayName", "1.0");
    if (isColumnFields) {
        fieldsBoostMap.put(COLUMNS_NAME_FIELD, "1.0");
        fieldsBoostMap.put(SCHEMA_COLUMNS_NAME_FIELD, "1.0");
    }
    if (testExceptions) {
        when(jsonHelper.unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}")).thenThrow(new IOException());
    } else {
        when(jsonHelper.unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}")).thenReturn(fieldsBoostMap);
    }
    IndexSearchHighlightField indexSearchHighlightField = new IndexSearchHighlightField("displayName", 100, Arrays.asList("displayName", "displayName.stemmed", "displayName.ngrams"), 5);
    IndexSearchHighlightFields highlightFields = new IndexSearchHighlightFields(Collections.singletonList(indexSearchHighlightField));
    IndexSearchHighlightField indexSearchHighlightFieldColumnOnly = new IndexSearchHighlightField("columnName", 100, Arrays.asList("columnName", "columnName.stemmed", "columnName.ngrams"), 5);
    IndexSearchHighlightFields highlightFieldsColumnOnly = new IndexSearchHighlightFields(Collections.singletonList(indexSearchHighlightFieldColumnOnly));
    if (testExceptions) {
        when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsConfigValue)).thenThrow(new IOException());
    } else {
        if (match != null && match.contains(MATCH_COLUMN)) {
            when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsColumnMatchConfigValue)).thenReturn(highlightFieldsColumnOnly);
        } else {
            when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsConfigValue)).thenReturn(highlightFields);
        }
    }
    when(searchRequestBuilder.setSource(any(SearchSourceBuilder.class))).thenReturn(searchRequestBuilderWithSource);
    when(searchRequestBuilderWithSource.setSize(SEARCH_RESULT_SIZE)).thenReturn(searchRequestBuilderWithSize);
    when(searchRequestBuilderWithSize.addSort(any(SortBuilder.class))).thenReturn(searchRequestBuilderWithSorting);
    when(searchRequestBuilderWithSorting.highlighter(any(HighlightBuilder.class))).thenReturn(searchRequestBuilderWithHighlighting);
    when(searchRequestBuilder.execute()).thenReturn(listenableActionFuture);
    when(listenableActionFuture.actionGet()).thenReturn(searchResponse);
    when(searchResponse.getHits()).thenReturn(searchHits);
    when(searchHits.hits()).thenReturn(searchHitArray);
    Map<String, Object> sourceMap1 = new HashMap<>();
    Map<String, Object> tagTypeMap = new HashMap<>();
    tagTypeMap.put(CODE, TAG_TYPE_CODE);
    sourceMap1.put(TAG_TYPE, tagTypeMap);
    when(searchHit1.sourceAsMap()).thenReturn(sourceMap1);
    Map<String, Object> sourceMap2 = new HashMap<>();
    Map<String, Object> businessObjectDefinitionMap = new HashMap<>();
    businessObjectDefinitionMap.put(CODE, NAMESPACE_CODE);
    sourceMap2.put(NAMESPACE, businessObjectDefinitionMap);
    when(searchHit2.sourceAsMap()).thenReturn(sourceMap2);
    when(searchHit1.getShard()).thenReturn(searchShardTarget1);
    when(searchHit2.getShard()).thenReturn(searchShardTarget2);
    when(searchShardTarget1.getIndex()).thenReturn(TAG_SEARCH_INDEX_NAME);
    when(searchShardTarget2.getIndex()).thenReturn(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME);
    when(searchHits.getTotalHits()).thenReturn(200L);
    Map<String, HighlightField> highlightFieldMap = new HashMap<>();
    highlightFieldMap.put("displayName", highlightField);
    when(searchHit1.getHighlightFields()).thenReturn(highlightFieldMap);
    when(searchHit2.getHighlightFields()).thenReturn(highlightFieldMap);
    // Create index search request
    final IndexSearchRequest indexSearchRequest = new IndexSearchRequest(SEARCH_TERM, searchFilters, facetList, isHitHighlightingEnabled);
    List<TagTypeIndexSearchResponseDto> tagTypeIndexSearchResponseDtos = Collections.singletonList(new TagTypeIndexSearchResponseDto("code", Collections.singletonList(new TagIndexSearchResponseDto("tag1", 1, null)), null));
    List<ResultTypeIndexSearchResponseDto> resultTypeIndexSearchResponseDto = Collections.singletonList(new ResultTypeIndexSearchResponseDto("type", 1, null));
    when(elasticsearchHelper.getNestedTagTagIndexSearchResponseDto(searchResponse)).thenReturn(tagTypeIndexSearchResponseDtos);
    when(elasticsearchHelper.getResultTypeIndexSearchResponseDto(searchResponse)).thenReturn(resultTypeIndexSearchResponseDto);
    when(elasticsearchHelper.getFacetsResponse(any(ElasticsearchResponseDto.class), eq(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME), eq(TAG_SEARCH_INDEX_NAME))).thenCallRealMethod();
    when(elasticsearchHelper.addIndexSearchFilterBooleanClause(any(), any(), any())).thenCallRealMethod();
    when(elasticsearchHelper.addFacetFieldAggregations(any(), any(SearchRequestBuilder.class))).thenReturn(searchRequestBuilder);
    when(herdSearchQueryHelper.determineNegationTermsPresent(any(IndexSearchRequest.class))).thenCallRealMethod();
    when(herdSearchQueryHelper.extractNegationTerms(any(IndexSearchRequest.class))).thenCallRealMethod();
    when(herdSearchQueryHelper.extractSearchPhrase(any(IndexSearchRequest.class))).thenCallRealMethod();
    SearchResult searchResult = mock(SearchResult.class);
    when(jestClientHelper.searchExecute(any())).thenReturn(searchResult);
    List<SearchResult.Hit<Map, Void>> searchHitList = new ArrayList<>();
    Gson gson = new Gson();
    Map<String, Object> map = new HashMap<>();
    map.put(DISPLAY_NAME_SOURCE, "Display Name");
    Map<String, Object> codeMap = new HashMap<>();
    codeMap.put(CODE, "Code");
    map.put(TAG_TYPE, codeMap);
    map.put(NAMESPACE, codeMap);
    JsonElement element = gson.toJsonTree(map);
    List<String> highlightList = new ArrayList<>();
    highlightList.add("Highlight 1");
    Map<String, List<String>> highlightMap = new HashMap<>();
    highlightMap.put("field", highlightList);
    SearchResult.Hit<Map, Void> hit1 = new SearchResult(gson).new Hit(HashMap.class, element, HashMap.class, null, highlightMap, null, setInvalidSearchResultIndexName ? INVALID_VALUE : TAG_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME, "type", 1.0);
    SearchResult.Hit<Map, Void> hit2 = new SearchResult(gson).new Hit(HashMap.class, element, HashMap.class, null, highlightMap, null, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, "type", 2.0);
    searchHitList.add(hit1);
    searchHitList.add(hit2);
    when(searchResult.getHits(Map.class)).thenReturn(searchHitList);
    when(searchResult.getTotal()).thenReturn(200L);
    MetricAggregation metricAggregation = mock(MetricAggregation.class);
    TermsAggregation termsAggregation = mock(TermsAggregation.class);
    when(searchResult.getAggregations()).thenReturn(metricAggregation);
    when(metricAggregation.getTermsAggregation(TAG_TYPE_FACET_AGGS)).thenReturn(termsAggregation);
    List<TermsAggregation.Entry> buckets = new ArrayList<>();
    TermsAggregation.Entry entry1 = mock(TermsAggregation.Entry.class);
    TermsAggregation.Entry entry2 = mock(TermsAggregation.Entry.class);
    buckets.add(entry1);
    buckets.add(entry2);
    when(termsAggregation.getBuckets()).thenReturn(buckets);
    // Call the method under test
    IndexSearchResponse indexSearchResponse = indexSearchDao.indexSearch(indexSearchRequest, fields, match, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME);
    List<IndexSearchResult> indexSearchResults = indexSearchResponse.getIndexSearchResults();
    assertThat("Index search results list is null.", indexSearchResults, not(nullValue()));
    assertThat(indexSearchResponse.getTotalIndexSearchResults(), is(200L));
    // Verify external method calls.
    verify(herdSearchQueryHelper).determineNegationTermsPresent(indexSearchRequest);
    verify(herdSearchQueryHelper).extractNegationTerms(indexSearchRequest);
    verify(herdSearchQueryHelper).extractSearchPhrase(indexSearchRequest);
    verify(configurationHelper).getProperty(ConfigurationValue.TAG_SHORT_DESCRIPTION_LENGTH, Integer.class);
    verify(configurationHelper).getProperty(ConfigurationValue.BUSINESS_OBJECT_DEFINITION_SHORT_DESCRIPTION_LENGTH, Integer.class);
    verify(configurationHelper, times(3)).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_STEMMED);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_NGRAMS);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_SHINGLES);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class);
    verify(configurationHelper, times(2)).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class);
    verify(configurationHelper, times(5)).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_SLOP, Integer.class);
    verify(jsonHelper, times(5)).unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}");
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getIndexSearchFilters())) {
        verify(elasticsearchHelper).addIndexSearchFilterBooleanClause(indexSearchRequest.getIndexSearchFilters(), BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME);
    }
    if (indexSearchRequest.isEnableHitHighlighting() != null) {
        verifyHitHighlightingInteractions(searchRequestBuilder, indexSearchRequest.isEnableHitHighlighting(), match);
    }
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        verify(elasticsearchHelper).addFacetFieldAggregations(any(), any(SearchRequestBuilder.class));
        if (indexSearchRequest.getFacetFields().contains(ElasticsearchHelper.TAG_FACET)) {
            verify(elasticsearchHelper).getNestedTagTagIndexSearchResponseDto(searchResult);
            verify(elasticsearchHelper).getTagTagIndexSearchResponseDto(searchResult);
        }
        if (indexSearchRequest.getFacetFields().contains(ElasticsearchHelper.RESULT_TYPE_FACET)) {
            verify(elasticsearchHelper).getResultTypeIndexSearchResponseDto(searchResult);
        }
        verify(elasticsearchHelper).getFacetsResponse(any(ElasticsearchResponseDto.class), eq(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME), eq(TAG_SEARCH_INDEX_NAME));
    }
    verify(jestClientHelper).searchExecute(any());
    verify(searchResult).getTotal();
    verify(searchResult).getHits(Map.class);
    verifyNoMoreInteractions(searchRequestBuilder, searchRequestBuilderWithSource, searchRequestBuilderWithSize, searchRequestBuilderWithSorting, searchRequestBuilderWithHighlighting, searchResponse, searchHits, searchHit1, searchHit2, searchShardTarget1, searchShardTarget2, highlightField, listenableActionFuture, searchResult, metricAggregation, termsAggregation, entry1, entry2);
    verifyNoMoreInteractionsHelper();
}

Example 2 with IndexSearchHighlightFields

use of org.finra.herd.model.dto.IndexSearchHighlightFields in project herd by FINRAOS.

the class IndexSearchDaoImpl method buildHighlightQuery.

/**
 * Builds a {@link HighlightBuilder} based on (pre/post)tags and fields fetched from the DB config which is added to the main {@link SearchRequestBuilder}
 *
 * @param preTag The specified pre-tag to be used for highlighting
 * @param postTag The specified post-tag to be used for highlighting
 * @param match the set of match fields that are to be searched upon in the index search
 *
 * @return A configured {@link HighlightBuilder} object
 */
private HighlightBuilder buildHighlightQuery(String preTag, String postTag, Set<String> match) {
    HighlightBuilder highlightBuilder = new HighlightBuilder();
    // Field matching is not needed since we are matching on multiple 'type' fields like stemmed and ngrams and enabling highlighting on all those fields
    // will yield duplicates
    highlightBuilder.requireFieldMatch(false);
    // Set the configured value for pre-tags for highlighting
    highlightBuilder.preTags(preTag);
    // Set the configured value for post-tags for highlighting
    highlightBuilder.postTags(postTag);
    // Get highlight fields value from configuration
    String highlightFieldsValue;
    // If the match column is included
    if (match != null && match.contains(MATCH_COLUMN)) {
        highlightFieldsValue = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_COLUMN_MATCH_HIGHLIGHT_FIELDS);
    } else {
        highlightFieldsValue = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_FIELDS);
    }
    try {
        @SuppressWarnings("unchecked") IndexSearchHighlightFields highlightFieldsConfig = jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsValue);
        highlightFieldsConfig.getHighlightFields().forEach(highlightFieldConfig -> {
            // set the field name to the configured value
            HighlightBuilder.Field highlightField = new HighlightBuilder.Field(highlightFieldConfig.getFieldName());
            // set matched_fields to the configured list of fields, this accounts for 'multifields' that analyze the same string in different ways
            if (CollectionUtils.isNotEmpty(highlightFieldConfig.getMatchedFields())) {
                highlightField.matchedFields(highlightFieldConfig.getMatchedFields().toArray(new String[0]));
            }
            // set fragment size to the configured value
            if (highlightFieldConfig.getFragmentSize() != null) {
                highlightField.fragmentSize(highlightFieldConfig.getFragmentSize());
            }
            // set the number of desired fragments to the configured value
            if (highlightFieldConfig.getNumOfFragments() != null) {
                highlightField.numOfFragments(highlightFieldConfig.getNumOfFragments());
            }
            highlightBuilder.field(highlightField);
        });
    } catch (IOException e) {
        LOGGER.warn("Could not parse the configured value for highlight fields: {}", highlightFieldsValue, e);
    }
    return highlightBuilder;
}