Examples with ElasticsearchRestClientException org.finra.herd.dao.exception.ElasticsearchRestClientException used on opensource projects

Example 1 with ElasticsearchRestClientException

use of org.finra.herd.dao.exception.ElasticsearchRestClientException in project herd by FINRAOS.

the class IndexSearchDaoImpl method indexSearch.

@Override
public IndexSearchResponse indexSearch(final IndexSearchRequest indexSearchRequest, final Set<String> fields, final Set<String> match, final String bdefActiveIndex, final String tagActiveIndex) {
    // Build a basic Boolean query upon which add all the necessary clauses as needed
    BoolQueryBuilder indexSearchQueryBuilder = QueryBuilders.boolQuery();
    String searchPhrase = indexSearchRequest.getSearchTerm();
    // If there is a search phrase, then process it
    if (StringUtils.isNotEmpty(searchPhrase)) {
        // Determine if negation terms are present
        boolean negationTermsExist = herdSearchQueryHelper.determineNegationTermsPresent(indexSearchRequest);
        // Add the negation queries builder within a 'must-not' clause to the parent bool query if negation terms exist
        if (negationTermsExist) {
            // Build negation queries- each term is added to the query with a 'must-not' clause,
            List<String> negationTerms = herdSearchQueryHelper.extractNegationTerms(indexSearchRequest);
            if (CollectionUtils.isNotEmpty(negationTerms)) {
                negationTerms.forEach(term -> indexSearchQueryBuilder.mustNot(buildMultiMatchQuery(term, PHRASE, 100f, FIELD_TYPE_STEMMED, match)));
            }
            // Remove the negation terms from the search phrase
            searchPhrase = herdSearchQueryHelper.extractSearchPhrase(indexSearchRequest);
        }
        // Build a Dismax query with three primary components (multi-match queries) with boost values, these values can be configured in the
        // DB which provides a way to dynamically tune search behavior at runtime:
        // 1. Phrase match query on shingles fields.
        // 2. Phrase prefix query on stemmed fields.
        // 3. Best fields query on ngrams fields.
        final MultiMatchQueryBuilder phrasePrefixMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE_PREFIX, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class), FIELD_TYPE_STEMMED, match);
        final MultiMatchQueryBuilder bestFieldsMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, BEST_FIELDS, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class), FIELD_TYPE_NGRAMS, match);
        final MultiMatchQueryBuilder phraseMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class), FIELD_TYPE_SHINGLES, match);
        final MultiMatchQueryBuilder phraseStemmedMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class), FIELD_TYPE_STEMMED, match);
        // Add the multi match queries to a dis max query and add to the parent bool query within a 'must' clause
        indexSearchQueryBuilder.must(disMaxQuery().add(phrasePrefixMultiMatchQueryBuilder).add(bestFieldsMultiMatchQueryBuilder).add(phraseMultiMatchQueryBuilder).add(phraseStemmedMultiMatchQueryBuilder));
    }
    // Add filter clauses if index search filters are specified in the request
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getIndexSearchFilters())) {
        indexSearchQueryBuilder.filter(elasticsearchHelper.addIndexSearchFilterBooleanClause(indexSearchRequest.getIndexSearchFilters(), bdefActiveIndex, tagActiveIndex));
    }
    // Get function score query builder
    FunctionScoreQueryBuilder functionScoreQueryBuilder = getFunctionScoreQueryBuilder(indexSearchQueryBuilder, bdefActiveIndex);
    // The fields in the search indexes to return
    final String[] searchSources = { NAME_SOURCE, NAMESPACE_CODE_SOURCE, TAG_CODE_SOURCE, TAG_TYPE_CODE_SOURCE, DISPLAY_NAME_SOURCE, DESCRIPTION_SOURCE, BDEF_TAGS_SOURCE, BDEF_TAGS_SEARCH_SCORE_MULTIPLIER };
    // Create a new indexSearch source builder
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    // Fetch only the required fields
    searchSourceBuilder.fetchSource(searchSources, null);
    searchSourceBuilder.query(functionScoreQueryBuilder);
    // Create a indexSearch request builder
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(bdefActiveIndex, tagActiveIndex);
    searchRequestBuilder.setSource(searchSourceBuilder).setSize(SEARCH_RESULT_SIZE).addSort(SortBuilders.scoreSort());
    // Add highlighting if specified in the request
    if (BooleanUtils.isTrue(indexSearchRequest.isEnableHitHighlighting())) {
        // Fetch configured 'tag' values for highlighting
        String preTag = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_PRETAGS);
        String postTag = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_POSTTAGS);
        searchRequestBuilder.highlighter(buildHighlightQuery(preTag, postTag, match));
    }
    // Add facet aggregations if specified in the request
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        searchRequestBuilder = elasticsearchHelper.addFacetFieldAggregations(new HashSet<>(indexSearchRequest.getFacetFields()), searchRequestBuilder);
    }
    // Log the actual elasticsearch query when debug is enabled
    LOGGER.debug("indexSearchRequest={}", searchRequestBuilder.toString());
    // Build the search request.
    SearchRequest searchRequest = new SearchRequest();
    searchRequest.source(searchSourceBuilder);
    searchRequest.indices(bdefActiveIndex, tagActiveIndex);
    // Create a search response object.
    SearchResponse searchResponse;
    // Get the Elasticsearch REST high level client. The REST high level client is auto closeable, so use try with resources.
    try (final RestHighLevelClient restHighLevelClient = elasticsearchRestHighLevelClientFactory.getRestHighLevelClient()) {
        // Retrieve the indexSearch response
        searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
    } catch (final IOException ioException) {
        LOGGER.error("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
        throw new ElasticsearchRestClientException("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
    }
    final List<IndexSearchResult> indexSearchResults = buildIndexSearchResults(fields, tagActiveIndex, bdefActiveIndex, searchResponse, indexSearchRequest.isEnableHitHighlighting());
    List<Facet> facets = null;
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        // Extract facets from the search response
        facets = new ArrayList<>(extractFacets(indexSearchRequest, searchResponse, bdefActiveIndex, tagActiveIndex));
    }
    // Get the total index search results in a null safe way.
    SearchHits searchHits = searchResponse.getHits();
    TotalHits totalHits = searchHits.getTotalHits();
    long totalIndexSearchResults = totalHits != null ? totalHits.value : 0L;
    return new IndexSearchResponse(totalIndexSearchResults, indexSearchResults, facets);
}

Example 2 with ElasticsearchRestClientException

use of org.finra.herd.dao.exception.ElasticsearchRestClientException in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method isValidDocumentIndex.

@Override
public boolean isValidDocumentIndex(final String indexName, final String id, final String json) {
    LOGGER.info("Validating Index Document, indexName={}, id={}.", indexName, id);
    // Build the get request.
    GetRequest getRequest = new GetRequest(indexName, id);
    // Create the get response object.
    GetResponse getResponse;
    // Get the Elasticsearch REST high level client. The REST high level client is auto closeable, so use try with resources.
    try (final RestHighLevelClient restHighLevelClient = elasticsearchRestHighLevelClientFactory.getRestHighLevelClient()) {
        // Make the get request.
        getResponse = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
    } catch (final IOException ioException) {
        LOGGER.error("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
        throw new ElasticsearchRestClientException("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
    }
    // Retrieve the JSON string from the get response
    final String jsonStringFromIndex = getResponse.getSourceAsString();
    // Return true if the json from the index is not null or empty and the json from the index matches the object from the database
    return StringUtils.isNotEmpty(jsonStringFromIndex) && jsonStringFromIndex.equals(json);
}

Example 3 with ElasticsearchRestClientException

use of org.finra.herd.dao.exception.ElasticsearchRestClientException in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method validateDocumentIndex.

@Override
public final void validateDocumentIndex(final String indexName, final String id, final String json) {
    LOGGER.info("Validating Elasticsearch document, indexName={}, id={}.", indexName, id);
    // Build the get request.
    GetRequest getRequest = new GetRequest(indexName, id);
    // Create the get response object.
    GetResponse getResponse;
    // Get the Elasticsearch REST high level client. The REST high level client is auto closeable, so use try with resources.
    try (final RestHighLevelClient restHighLevelClient = elasticsearchRestHighLevelClientFactory.getRestHighLevelClient()) {
        // Make the get request.
        getResponse = restHighLevelClient.get(getRequest, RequestOptions.DEFAULT);
    } catch (final IOException ioException) {
        LOGGER.error("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
        throw new ElasticsearchRestClientException("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
    }
    // Retrieve the JSON string from the get response
    final String jsonStringFromIndex = getResponse.getSourceAsString();
    // If the document does not exist in the index add the document to the index
    if (StringUtils.isEmpty(jsonStringFromIndex)) {
        LOGGER.warn("Document does not exist in the index, adding the document to the index.");
        // Create the index document.
        createIndexDocument(indexName, id, json);
    } else // Else if the JSON does not match the JSON from the index update the index
    if (!json.equals(jsonStringFromIndex)) {
        LOGGER.warn("Document does not match the document in the index, updating the document in the index.");
        // Note that create index will update the document if it exists.
        createIndexDocument(indexName, id, json);
    }
}

Example 4 with ElasticsearchRestClientException

use of org.finra.herd.dao.exception.ElasticsearchRestClientException in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method deleteIndexDocuments.

@Override
public final void deleteIndexDocuments(final String indexName, final List<Long> ids) {
    LOGGER.info("Deleting Elasticsearch documents from index, indexName={}, ids={}.", indexName, ids.stream().map(Object::toString).collect(Collectors.joining(",")));
    List<String> allIndices = getAliases(indexName);
    // Get the Elasticsearch REST high level client. The REST high level client is auto closeable, so use try with resources.
    try (final RestHighLevelClient restHighLevelClient = elasticsearchRestHighLevelClientFactory.getRestHighLevelClient()) {
        // For each index in the list of indices.
        for (String index : allIndices) {
            // Prepare a bulk request
            BulkRequest bulkRequest = new BulkRequest();
            // For each document prepare an insert request and add it to the bulk request
            ids.forEach(id -> bulkRequest.add(new DeleteRequest(index, id.toString())));
            BulkResponse bulkResponse = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
            // If there are failures log them
            if (bulkResponse.hasFailures()) {
                LOGGER.error("Bulk response error={}.", bulkResponse.buildFailureMessage());
            }
        }
    } catch (final IOException ioException) {
        LOGGER.error("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
        throw new ElasticsearchRestClientException("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
    }
}

Example 5 with ElasticsearchRestClientException

use of org.finra.herd.dao.exception.ElasticsearchRestClientException in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method getNumberOfTypesInIndex.

@Override
public long getNumberOfTypesInIndex(final String indexName) {
    LOGGER.info("Counting the number of documents in index={}.", indexName);
    // Build the count request.
    CountRequest countRequest = new CountRequest(indexName);
    countRequest.query(QueryBuilders.matchAllQuery());
    // Create the count response object.
    CountResponse countResponse;
    // Get the Elasticsearch REST high level client. The REST high level client is auto closeable, so use try with resources.
    try (final RestHighLevelClient restHighLevelClient = elasticsearchRestHighLevelClientFactory.getRestHighLevelClient()) {
        // Make the count request.
        countResponse = restHighLevelClient.count(countRequest, RequestOptions.DEFAULT);
    } catch (final IOException ioException) {
        LOGGER.error("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
        throw new ElasticsearchRestClientException("Caught IOException while attempting to use the ElasticsearchRestHighLevelClient.", ioException);
    }
    return countResponse.getCount();
}