Examples with ElasticsearchClientImpl org.finra.herd.dao.helper.ElasticsearchClientImpl used on opensource projects

Example 1 with ElasticsearchClientImpl

use of org.finra.herd.dao.helper.ElasticsearchClientImpl in project herd by FINRAOS.

the class BusinessObjectDefinitionIndexSearchDaoImpl method findAllBusinessObjectDefinitions.

@Override
public ElasticsearchResponseDto findAllBusinessObjectDefinitions(String indexName, String documentType, Set<String> facetFieldsList) {
    LOGGER.info("Elasticsearch get all business object definition documents from index, indexName={} and documentType={}.", indexName, documentType);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.fetchSource(new String[] { DATA_PROVIDER_NAME_SOURCE, DESCRIPTION_SOURCE, DISPLAY_NAME_SOURCE, NAME_SOURCE, NAMESPACE_CODE_SOURCE }, null);
    ElasticsearchResponseDto elasticsearchResponseDto = new ElasticsearchResponseDto();
    // Create a search request and set the scroll time and scroll size
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(indexName);
    searchRequestBuilder.setTypes(documentType).setSource(searchSourceBuilder).addSort(SortBuilders.fieldSort(BUSINESS_OBJECT_DEFINITION_SORT_FIELD).order(SortOrder.ASC)).addSort(SortBuilders.fieldSort(NAMESPACE_CODE_SORT_FIELD).order(SortOrder.ASC));
    // Add aggregation builder if facet fields are present
    addFacetFieldAggregations(facetFieldsList, elasticsearchResponseDto, searchRequestBuilder, indexName);
    elasticsearchResponseDto.setBusinessObjectDefinitionIndexSearchResponseDtos(scrollSearchResultsIntoBusinessObjectDefinitionDto(searchRequestBuilder, indexName));
    return elasticsearchResponseDto;
}

Example 2 with ElasticsearchClientImpl

use of org.finra.herd.dao.helper.ElasticsearchClientImpl in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method getIdsInIndex.

/**
 * The ids in index function will take as arguments the index name and the document type and will return a list of all the ids in the index.
 */
@Override
public final List<String> getIdsInIndex(String indexName, String documentType) {
    // Create an array list for storing the ids
    List<String> idList = new ArrayList<>();
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.query(QueryBuilders.matchAllQuery());
    // Create a search request and set the scroll time and scroll size
    final SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(indexName).setTypes(documentType).setScroll(new TimeValue(ELASTIC_SEARCH_SCROLL_KEEP_ALIVE_TIME)).setSize(ELASTIC_SEARCH_SCROLL_PAGE_SIZE).setSource(searchSourceBuilder);
    // Retrieve the search response
    final Search.Builder searchBuilder = new Search.Builder(searchRequestBuilder.toString()).addIndex(indexName);
    searchBuilder.setParameter(Parameters.SIZE, ELASTIC_SEARCH_SCROLL_PAGE_SIZE);
    searchBuilder.setParameter(Parameters.SCROLL, new TimeValue(ELASTIC_SEARCH_SCROLL_KEEP_ALIVE_TIME).toString());
    JestResult jestResult = jestClientHelper.searchExecute(searchBuilder.build());
    // While there are hits available, page through the results and add them to the id list
    while (jestResult.getSourceAsStringList().size() != 0) {
        for (String jsonString : jestResult.getSourceAsStringList()) {
            JsonElement root = new JsonParser().parse(jsonString);
            idList.add(root.getAsJsonObject().get("id").getAsString());
        }
        String scrollId = jestResult.getJsonObject().get("_scroll_id").getAsString();
        SearchScroll scroll = new SearchScroll.Builder(scrollId, new TimeValue(ELASTIC_SEARCH_SCROLL_KEEP_ALIVE_TIME).toString()).build();
        jestResult = jestClientHelper.searchScrollExecute(scroll);
    }
    return idList;
}

Example 3 with ElasticsearchClientImpl

use of org.finra.herd.dao.helper.ElasticsearchClientImpl in project herd by FINRAOS.

the class IndexSearchDaoImpl method indexSearch.

@Override
public IndexSearchResponse indexSearch(final IndexSearchRequest indexSearchRequest, final Set<String> fields, final Set<String> match, final String bdefActiveIndex, final String tagActiveIndex) {
    boolean negationTermsExist = herdSearchQueryHelper.determineNegationTermsPresent(indexSearchRequest);
    // Build a basic Boolean query upon which add all the necessary clauses as needed
    BoolQueryBuilder indexSearchQueryBuilder = QueryBuilders.boolQuery();
    String searchPhrase = indexSearchRequest.getSearchTerm();
    // Add the negation queries builder within a 'must-not' clause to the parent bool query if negation terms exist
    if (negationTermsExist) {
        // Build negation queries- each term is added to the query with a 'must-not' clause,
        List<String> negationTerms = herdSearchQueryHelper.extractNegationTerms(indexSearchRequest);
        if (CollectionUtils.isNotEmpty(negationTerms)) {
            negationTerms.forEach(term -> {
                indexSearchQueryBuilder.mustNot(buildMultiMatchQuery(term, PHRASE, 100f, FIELD_TYPE_STEMMED, match));
            });
        }
        // Remove the negation terms from the search phrase
        searchPhrase = herdSearchQueryHelper.extractSearchPhrase(indexSearchRequest);
    }
    // Build a Dismax query with three primary components (multi-match queries) with boost values, these values can be configured in the
    // DB which provides a way to dynamically tune search behavior at runtime:
    // 1. Phrase match query on shingles fields.
    // 2. Phrase prefix query on stemmed fields.
    // 3. Best fields query on ngrams fields.
    final MultiMatchQueryBuilder phrasePrefixMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE_PREFIX, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class), FIELD_TYPE_STEMMED, match);
    final MultiMatchQueryBuilder bestFieldsMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, BEST_FIELDS, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class), FIELD_TYPE_NGRAMS, match);
    final MultiMatchQueryBuilder phraseMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class), FIELD_TYPE_SHINGLES, match);
    final MultiMatchQueryBuilder phraseStemmedMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class), FIELD_TYPE_STEMMED, match);
    // Add the multi match queries to a dis max query and add to the parent bool query within a 'must' clause
    indexSearchQueryBuilder.must(disMaxQuery().add(phrasePrefixMultiMatchQueryBuilder).add(bestFieldsMultiMatchQueryBuilder).add(phraseMultiMatchQueryBuilder).add(phraseStemmedMultiMatchQueryBuilder));
    // Add filter clauses if index search filters are specified in the request
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getIndexSearchFilters())) {
        indexSearchQueryBuilder.filter(elasticsearchHelper.addIndexSearchFilterBooleanClause(indexSearchRequest.getIndexSearchFilters(), bdefActiveIndex, tagActiveIndex));
    }
    // Get function score query builder
    FunctionScoreQueryBuilder functionScoreQueryBuilder = getFunctionScoreQueryBuilder(indexSearchQueryBuilder, bdefActiveIndex);
    // The fields in the search indexes to return
    final String[] searchSources = { NAME_SOURCE, NAMESPACE_CODE_SOURCE, TAG_CODE_SOURCE, TAG_TYPE_CODE_SOURCE, DISPLAY_NAME_SOURCE, DESCRIPTION_SOURCE, BDEF_TAGS_SOURCE, BDEF_TAGS_SEARCH_SCORE_MULTIPLIER };
    // Create a new indexSearch source builder
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    // Fetch only the required fields
    searchSourceBuilder.fetchSource(searchSources, null);
    searchSourceBuilder.query(functionScoreQueryBuilder);
    // Create a indexSearch request builder
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(bdefActiveIndex, tagActiveIndex);
    searchRequestBuilder.setSource(searchSourceBuilder).setSize(SEARCH_RESULT_SIZE).addSort(SortBuilders.scoreSort());
    // Add highlighting if specified in the request
    if (BooleanUtils.isTrue(indexSearchRequest.isEnableHitHighlighting())) {
        // Fetch configured 'tag' values for highlighting
        String preTag = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_PRETAGS);
        String postTag = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_POSTTAGS);
        searchRequestBuilder.highlighter(buildHighlightQuery(preTag, postTag, match));
    }
    // Add facet aggregations if specified in the request
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        searchRequestBuilder = elasticsearchHelper.addFacetFieldAggregations(new HashSet<>(indexSearchRequest.getFacetFields()), searchRequestBuilder);
    }
    // Log the actual elasticsearch query when debug is enabled
    LOGGER.debug("indexSearchRequest={}", searchRequestBuilder.toString());
    // Retrieve the indexSearch response
    final Search.Builder searchBuilder = new Search.Builder(searchRequestBuilder.toString()).addIndices(Arrays.asList(bdefActiveIndex, tagActiveIndex));
    final SearchResult searchResult = jestClientHelper.searchExecute(searchBuilder.build());
    final List<IndexSearchResult> indexSearchResults = buildIndexSearchResults(fields, tagActiveIndex, bdefActiveIndex, searchResult, indexSearchRequest.isEnableHitHighlighting());
    List<Facet> facets = null;
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        // Extract facets from the search response
        facets = new ArrayList<>(extractFacets(indexSearchRequest, searchResult, bdefActiveIndex, tagActiveIndex));
    }
    return new IndexSearchResponse(searchResult.getTotal(), indexSearchResults, facets);
}

Example 4 with ElasticsearchClientImpl

use of org.finra.herd.dao.helper.ElasticsearchClientImpl in project herd by FINRAOS.

the class BusinessObjectDefinitionIndexSearchDaoImpl method searchBusinessObjectDefinitionsByTags.

@Override
public ElasticsearchResponseDto searchBusinessObjectDefinitionsByTags(String indexName, String documentType, List<Map<SearchFilterType, List<TagEntity>>> nestedTagEntityMaps, Set<String> facetFieldsList) {
    ElasticsearchResponseDto elasticsearchResponseDto = new ElasticsearchResponseDto();
    List<List<TagEntity>> nestedInclusionTagEntityLists = new ArrayList<>();
    List<List<TagEntity>> nestedExclusionTagEntityLists = new ArrayList<>();
    for (Map<SearchFilterType, List<TagEntity>> tagEntityMap : nestedTagEntityMaps) {
        if (tagEntityMap.containsKey(SearchFilterType.INCLUSION_SEARCH_FILTER)) {
            nestedInclusionTagEntityLists.add(tagEntityMap.get(SearchFilterType.INCLUSION_SEARCH_FILTER));
        } else if (tagEntityMap.containsKey(SearchFilterType.EXCLUSION_SEARCH_FILTER)) {
            nestedExclusionTagEntityLists.add(tagEntityMap.get(SearchFilterType.EXCLUSION_SEARCH_FILTER));
        }
    }
    LOGGER.info("Searching Elasticsearch business object definition documents from index, indexName={} and documentType={}, by tagEntityList={}", indexName, documentType, tagEntityListToString(flattenTagEntitiesList(nestedInclusionTagEntityLists)));
    LOGGER.info("Excluding the following tagEntityList={}", indexName, documentType, tagEntityListToString(flattenTagEntitiesList(nestedExclusionTagEntityLists)));
    BoolQueryBuilder compoundSearchFiltersQueryBuilder = new BoolQueryBuilder();
    // If there are only exclusion tag entities then, get everything else, but the exclusion tags.
    if (CollectionUtils.isEmpty(flattenTagEntitiesList(nestedInclusionTagEntityLists))) {
        WildcardQueryBuilder wildcardQueryBuilder = QueryBuilders.wildcardQuery(NAME_FIELD, "*");
        compoundSearchFiltersQueryBuilder.must(wildcardQueryBuilder);
    }
    // Inclusion
    for (List<TagEntity> tagEntities : nestedInclusionTagEntityLists) {
        BoolQueryBuilder searchFilterQueryBuilder = new BoolQueryBuilder();
        for (TagEntity tagEntity : tagEntities) {
            // Add constant-score term queries for tagType-code and tag-code from the tag-key.
            ConstantScoreQueryBuilder searchKeyQueryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(TAGTYPE_CODE_FIELD, tagEntity.getTagType().getCode())).must(QueryBuilders.termQuery(TAG_CODE_FIELD, tagEntity.getTagCode())));
            // Individual tag-keys are OR-ed
            searchFilterQueryBuilder.should(searchKeyQueryBuilder);
        }
        // Individual search-filters are AND-ed
        compoundSearchFiltersQueryBuilder.must(searchFilterQueryBuilder);
    }
    // Exclusion
    for (List<TagEntity> tagEntities : nestedExclusionTagEntityLists) {
        for (TagEntity tagEntity : tagEntities) {
            // Add constant-score term queries for tagType-code and tag-code from the tag-key.
            QueryBuilder searchKeyQueryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(TAGTYPE_CODE_FIELD, tagEntity.getTagType().getCode())).must(QueryBuilders.termQuery(TAG_CODE_FIELD, tagEntity.getTagCode()));
            // Exclusion: individual tag-keys are added as a must not query
            compoundSearchFiltersQueryBuilder.mustNot(searchKeyQueryBuilder);
        }
    }
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    // Fetch only the required fields
    searchSourceBuilder.fetchSource(new String[] { DATA_PROVIDER_NAME_SOURCE, DESCRIPTION_SOURCE, DISPLAY_NAME_SOURCE, NAME_SOURCE, NAMESPACE_CODE_SOURCE }, null);
    searchSourceBuilder.query(compoundSearchFiltersQueryBuilder);
    // Create a search request and set the scroll time and scroll size
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(indexName);
    // Construct scroll query
    searchRequestBuilder.setTypes(documentType).setScroll(new TimeValue(ELASTIC_SEARCH_SCROLL_KEEP_ALIVE_TIME)).setSize(ELASTIC_SEARCH_SCROLL_PAGE_SIZE).setSource(searchSourceBuilder).addSort(SortBuilders.fieldSort(BUSINESS_OBJECT_DEFINITION_SORT_FIELD).order(SortOrder.ASC)).addSort(SortBuilders.fieldSort(NAMESPACE_CODE_SORT_FIELD).order(SortOrder.ASC));
    // Add aggregation builder if facet fields are present
    if (CollectionUtils.isNotEmpty(facetFieldsList)) {
        addFacetFieldAggregations(facetFieldsList, elasticsearchResponseDto, searchRequestBuilder, indexName);
    }
    // Log the actual search query
    LOGGER.info("bdefIndexSearchQuery={}", searchRequestBuilder.toString());
    elasticsearchResponseDto.setBusinessObjectDefinitionIndexSearchResponseDtos(scrollSearchResultsIntoBusinessObjectDefinitionDto(searchRequestBuilder, indexName));
    return elasticsearchResponseDto;
}