Examples with InvalidSearchException org.apache.metron.indexing.dao.search.InvalidSearchException used on opensource projects

Example 1 with InvalidSearchException

use of org.apache.metron.indexing.dao.search.InvalidSearchException in project metron by apache.

the class SearchServiceImpl method search.

@Override
public SearchResponse search(SearchRequest searchRequest) throws RestException {
    try {
        if (searchRequest.getIndices() == null || searchRequest.getIndices().isEmpty()) {
            List<String> indices = getDefaultIndices();
            // metaalerts should be included by default in search requests
            indices.add(METAALERT_TYPE);
            searchRequest.setIndices(indices);
        }
        if (searchRequest.getFacetFields() != null && searchRequest.getFacetFields().isEmpty()) {
            searchRequest.setFacetFields(getDefaultFacetFields());
        }
        return dao.search(searchRequest);
    } catch (InvalidSearchException ise) {
        throw new RestException(ise.getMessage(), ise);
    }
}

Example 2 with InvalidSearchException

use of org.apache.metron.indexing.dao.search.InvalidSearchException in project metron by apache.

the class ElasticsearchDao method buildSearchRequest.

/**
 * Builds an Elasticsearch search request.
 * @param searchRequest The Metron search request.
 * @param queryBuilder
 * @return An Elasticsearch search request.
 */
private org.elasticsearch.action.search.SearchRequest buildSearchRequest(SearchRequest searchRequest, QueryBuilder queryBuilder) throws InvalidSearchException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Got search request; request={}", ElasticsearchUtils.toJSON(searchRequest).orElse("???"));
    }
    SearchSourceBuilder searchBuilder = new SearchSourceBuilder().size(searchRequest.getSize()).from(searchRequest.getFrom()).query(queryBuilder).trackScores(true);
    List<String> fields = searchRequest.getFields();
    // column metadata needed to understand the type of each sort field
    Map<String, FieldType> meta;
    try {
        meta = getColumnMetadata(searchRequest.getIndices());
    } catch (IOException e) {
        throw new InvalidSearchException("Unable to get column metadata", e);
    }
    // handle sort fields
    for (SortField sortField : searchRequest.getSort()) {
        // what type is the sort field?
        FieldType sortFieldType = meta.getOrDefault(sortField.getField(), FieldType.OTHER);
        // sort order - if ascending missing values sorted last. otherwise, missing values sorted first
        org.elasticsearch.search.sort.SortOrder sortOrder = getElasticsearchSortOrder(sortField.getSortOrder());
        String missingSortOrder;
        if (sortOrder == org.elasticsearch.search.sort.SortOrder.DESC) {
            missingSortOrder = SORT_MISSING_LAST;
        } else {
            missingSortOrder = SORT_MISSING_FIRST;
        }
        // sort by the field - missing fields always last
        FieldSortBuilder sortBy = new FieldSortBuilder(sortField.getField()).order(sortOrder).missing(missingSortOrder).unmappedType(sortFieldType.getFieldType());
        searchBuilder.sort(sortBy);
    }
    // handle search fields
    if (fields != null) {
        searchBuilder.fetchSource("*", null);
    } else {
        searchBuilder.fetchSource(true);
    }
    List<String> facetFields = searchRequest.getFacetFields();
    // handle facet fields
    if (facetFields != null) {
        // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html
        for (String field : facetFields) {
            String name = getFacetAggregationName(field);
            TermsAggregationBuilder terms = AggregationBuilders.terms(name).field(field);
            // new TermsBuilder(name).field(field);
            searchBuilder.aggregation(terms);
        }
    }
    // return the search request
    String[] indices = wildcardIndices(searchRequest.getIndices());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString());
    }
    return new org.elasticsearch.action.search.SearchRequest().indices(indices).source(searchBuilder);
}

Example 3 with InvalidSearchException

use of org.apache.metron.indexing.dao.search.InvalidSearchException in project metron by apache.

the class ElasticsearchDao method buildSearchResponse.

/**
 * Builds a search response.
 *
 * This effectively transforms an Elasticsearch search response into a Metron search response.
 *
 * @param searchRequest The Metron search request.
 * @param esResponse The Elasticsearch search response.
 * @return A Metron search response.
 * @throws InvalidSearchException
 */
private SearchResponse buildSearchResponse(SearchRequest searchRequest, org.elasticsearch.action.search.SearchResponse esResponse) throws InvalidSearchException {
    SearchResponse searchResponse = new SearchResponse();
    searchResponse.setTotal(esResponse.getHits().getTotalHits());
    // search hits --> search results
    List<SearchResult> results = new ArrayList<>();
    for (SearchHit hit : esResponse.getHits().getHits()) {
        results.add(getSearchResult(hit, searchRequest.getFields()));
    }
    searchResponse.setResults(results);
    // handle facet fields
    if (searchRequest.getFacetFields() != null) {
        List<String> facetFields = searchRequest.getFacetFields();
        Map<String, FieldType> commonColumnMetadata;
        try {
            commonColumnMetadata = getColumnMetadata(searchRequest.getIndices());
        } catch (IOException e) {
            throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(searchRequest.getIndices().toArray())));
        }
        searchResponse.setFacetCounts(getFacetCounts(facetFields, esResponse.getAggregations(), commonColumnMetadata));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Built search response; response={}", ElasticsearchUtils.toJSON(searchResponse).orElse("???"));
    }
    return searchResponse;
}

Example 4 with InvalidSearchException

use of org.apache.metron.indexing.dao.search.InvalidSearchException in project metron by apache.

the class ElasticsearchRequestSubmitter method submitSearch.

/**
 * Submit a search to Elasticsearch.
 * @param request A search request.
 * @return The search response.
 */
public SearchResponse submitSearch(SearchRequest request) throws InvalidSearchException {
    LOG.debug("About to submit a search; request={}", ElasticsearchUtils.toJSON(request).orElse("???"));
    // submit the search request
    org.elasticsearch.action.search.SearchResponse esResponse;
    try {
        esResponse = client.search(request).actionGet();
        LOG.debug("Got Elasticsearch response; response={}", esResponse.toString());
    } catch (SearchPhaseExecutionException e) {
        String msg = String.format("Failed to execute search; error='%s', search='%s'", ExceptionUtils.getRootCauseMessage(e), ElasticsearchUtils.toJSON(request).orElse("???"));
        LOG.error(msg, e);
        throw new InvalidSearchException(msg, e);
    }
    // check for shard failures
    if (esResponse.getFailedShards() > 0) {
        handleShardFailures(request, esResponse);
    }
    // validate the response status
    if (RestStatus.OK == esResponse.status()) {
        return esResponse;
    } else {
        // the search was not successful
        String msg = String.format("Bad search response; status=%s, timeout=%s, terminatedEarly=%s", esResponse.status(), esResponse.isTimedOut(), esResponse.isTerminatedEarly());
        LOG.error(msg);
        throw new InvalidSearchException(msg);
    }
}

Example 5 with InvalidSearchException

use of org.apache.metron.indexing.dao.search.InvalidSearchException in project metron by apache.

the class InMemoryDao method search.

@Override
public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException {
    if (config.getMaxSearchResults() != null && searchRequest.getSize() > config.getMaxSearchResults()) {
        throw new InvalidSearchException("Search result size must be less than " + config.getMaxSearchResults());
    }
    List<SearchResult> response = new ArrayList<>();
    for (String index : searchRequest.getIndices()) {
        String i = null;
        for (String storedIdx : BACKING_STORE.keySet()) {
            if (storedIdx.equals(index) || storedIdx.startsWith(index + "_")) {
                i = storedIdx;
            }
        }
        if (i == null) {
            continue;
        }
        for (String doc : BACKING_STORE.get(i)) {
            Map<String, Object> docParsed = parse(doc);
            if (isMatch(searchRequest.getQuery(), docParsed)) {
                SearchResult result = new SearchResult();
                result.setSource(docParsed);
                result.setScore((float) Math.random());
                result.setId(docParsed.getOrDefault(Constants.GUID, UUID.randomUUID()).toString());
                response.add(result);
            }
        }
    }
    if (searchRequest.getSort().size() != 0) {
        Collections.sort(response, sorted(searchRequest.getSort()));
    }
    SearchResponse ret = new SearchResponse();
    List<SearchResult> finalResp = new ArrayList<>();
    int maxSize = config.getMaxSearchResults() == null ? searchRequest.getSize() : config.getMaxSearchResults();
    for (int i = searchRequest.getFrom(); i < response.size() && finalResp.size() <= maxSize; ++i) {
        finalResp.add(response.get(i));
    }
    ret.setTotal(response.size());
    ret.setResults(finalResp);
    Map<String, Map<String, Long>> facetCounts = new HashMap<>();
    List<String> facetFields = searchRequest.getFacetFields();
    if (facetFields != null) {
        for (String facet : facetFields) {
            facetCounts.put(facet, FACET_COUNTS.get(facet));
        }
        ret.setFacetCounts(facetCounts);
    }
    return ret;
}