Examples with FieldType org.apache.metron.indexing.dao.search.FieldType used on opensource projects

Example 1 with FieldType

use of org.apache.metron.indexing.dao.search.FieldType in project metron by apache.

the class ElasticsearchDao method buildSearchRequest.

/**
 * Builds an Elasticsearch search request.
 * @param searchRequest The Metron search request.
 * @param queryBuilder
 * @return An Elasticsearch search request.
 */
private org.elasticsearch.action.search.SearchRequest buildSearchRequest(SearchRequest searchRequest, QueryBuilder queryBuilder) throws InvalidSearchException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Got search request; request={}", ElasticsearchUtils.toJSON(searchRequest).orElse("???"));
    }
    SearchSourceBuilder searchBuilder = new SearchSourceBuilder().size(searchRequest.getSize()).from(searchRequest.getFrom()).query(queryBuilder).trackScores(true);
    List<String> fields = searchRequest.getFields();
    // column metadata needed to understand the type of each sort field
    Map<String, FieldType> meta;
    try {
        meta = getColumnMetadata(searchRequest.getIndices());
    } catch (IOException e) {
        throw new InvalidSearchException("Unable to get column metadata", e);
    }
    // handle sort fields
    for (SortField sortField : searchRequest.getSort()) {
        // what type is the sort field?
        FieldType sortFieldType = meta.getOrDefault(sortField.getField(), FieldType.OTHER);
        // sort order - if ascending missing values sorted last. otherwise, missing values sorted first
        org.elasticsearch.search.sort.SortOrder sortOrder = getElasticsearchSortOrder(sortField.getSortOrder());
        String missingSortOrder;
        if (sortOrder == org.elasticsearch.search.sort.SortOrder.DESC) {
            missingSortOrder = SORT_MISSING_LAST;
        } else {
            missingSortOrder = SORT_MISSING_FIRST;
        }
        // sort by the field - missing fields always last
        FieldSortBuilder sortBy = new FieldSortBuilder(sortField.getField()).order(sortOrder).missing(missingSortOrder).unmappedType(sortFieldType.getFieldType());
        searchBuilder.sort(sortBy);
    }
    // handle search fields
    if (fields != null) {
        searchBuilder.fetchSource("*", null);
    } else {
        searchBuilder.fetchSource(true);
    }
    List<String> facetFields = searchRequest.getFacetFields();
    // handle facet fields
    if (facetFields != null) {
        // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html
        for (String field : facetFields) {
            String name = getFacetAggregationName(field);
            TermsAggregationBuilder terms = AggregationBuilders.terms(name).field(field);
            // new TermsBuilder(name).field(field);
            searchBuilder.aggregation(terms);
        }
    }
    // return the search request
    String[] indices = wildcardIndices(searchRequest.getIndices());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString());
    }
    return new org.elasticsearch.action.search.SearchRequest().indices(indices).source(searchBuilder);
}

Example 2 with FieldType

use of org.apache.metron.indexing.dao.search.FieldType in project metron by apache.

the class ElasticsearchDao method buildSearchResponse.

/**
 * Builds a search response.
 *
 * This effectively transforms an Elasticsearch search response into a Metron search response.
 *
 * @param searchRequest The Metron search request.
 * @param esResponse The Elasticsearch search response.
 * @return A Metron search response.
 * @throws InvalidSearchException
 */
private SearchResponse buildSearchResponse(SearchRequest searchRequest, org.elasticsearch.action.search.SearchResponse esResponse) throws InvalidSearchException {
    SearchResponse searchResponse = new SearchResponse();
    searchResponse.setTotal(esResponse.getHits().getTotalHits());
    // search hits --> search results
    List<SearchResult> results = new ArrayList<>();
    for (SearchHit hit : esResponse.getHits().getHits()) {
        results.add(getSearchResult(hit, searchRequest.getFields()));
    }
    searchResponse.setResults(results);
    // handle facet fields
    if (searchRequest.getFacetFields() != null) {
        List<String> facetFields = searchRequest.getFacetFields();
        Map<String, FieldType> commonColumnMetadata;
        try {
            commonColumnMetadata = getColumnMetadata(searchRequest.getIndices());
        } catch (IOException e) {
            throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(searchRequest.getIndices().toArray())));
        }
        searchResponse.setFacetCounts(getFacetCounts(facetFields, esResponse.getAggregations(), commonColumnMetadata));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Built search response; response={}", ElasticsearchUtils.toJSON(searchResponse).orElse("???"));
    }
    return searchResponse;
}

Example 3 with FieldType

use of org.apache.metron.indexing.dao.search.FieldType in project metron by apache.

the class InMemoryDao method getColumnMetadata.

@Override
public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
    Map<String, FieldType> indexColumnMetadata = new HashMap<>();
    for (String index : indices) {
        if (COLUMN_METADATA.containsKey(index)) {
            Map<String, FieldType> columnMetadata = COLUMN_METADATA.get(index);
            for (Entry entry : columnMetadata.entrySet()) {
                String field = (String) entry.getKey();
                FieldType type = (FieldType) entry.getValue();
                if (indexColumnMetadata.containsKey(field)) {
                    if (!type.equals(indexColumnMetadata.get(field))) {
                        indexColumnMetadata.put(field, FieldType.OTHER);
                    }
                } else {
                    indexColumnMetadata.put(field, type);
                }
            }
        }
    }
    return indexColumnMetadata;
}

Example 4 with FieldType

use of org.apache.metron.indexing.dao.search.FieldType in project metron by apache.

the class ElasticsearchColumnMetadataDao method getColumnMetadata.

@SuppressWarnings("unchecked")
@Override
public Map<String, FieldType> getColumnMetadata(List<String> indices) throws IOException {
    Map<String, FieldType> indexColumnMetadata = new HashMap<>();
    Map<String, String> previousIndices = new HashMap<>();
    Set<String> fieldBlackList = new HashSet<>();
    String[] latestIndices = getLatestIndices(indices);
    if (latestIndices.length > 0) {
        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = adminClient.indices().getMappings(new GetMappingsRequest().indices(latestIndices)).actionGet().getMappings();
        // for each index
        for (Object key : mappings.keys().toArray()) {
            String indexName = key.toString();
            ImmutableOpenMap<String, MappingMetaData> mapping = mappings.get(indexName);
            // for each mapping in the index
            Iterator<String> mappingIterator = mapping.keysIt();
            while (mappingIterator.hasNext()) {
                MappingMetaData mappingMetaData = mapping.get(mappingIterator.next());
                Map<String, Object> sourceAsMap = mappingMetaData.getSourceAsMap();
                if (sourceAsMap.containsKey("properties")) {
                    Map<String, Map<String, String>> map = (Map<String, Map<String, String>>) sourceAsMap.get("properties");
                    // for each field in the mapping
                    for (String field : map.keySet()) {
                        if (!fieldBlackList.contains(field)) {
                            FieldType type = toFieldType(map.get(field).get("type"));
                            if (!indexColumnMetadata.containsKey(field)) {
                                indexColumnMetadata.put(field, type);
                                // record the last index in which a field exists, to be able to print helpful error message on type mismatch
                                previousIndices.put(field, indexName);
                            } else {
                                FieldType previousType = indexColumnMetadata.get(field);
                                if (!type.equals(previousType)) {
                                    String previousIndexName = previousIndices.get(field);
                                    LOG.error(String.format("Field type mismatch: %s.%s has type %s while %s.%s has type %s.  Defaulting type to %s.", indexName, field, type.getFieldType(), previousIndexName, field, previousType.getFieldType(), FieldType.OTHER.getFieldType()));
                                    indexColumnMetadata.put(field, FieldType.OTHER);
                                    // the field is defined in multiple indices with different types; ignore the field as type has been set to OTHER
                                    fieldBlackList.add(field);
                                }
                            }
                        }
                    }
                }
            }
        }
    } else {
        LOG.info(String.format("Unable to find any latest indices; indices=%s", indices));
    }
    return indexColumnMetadata;
}

Example 5 with FieldType

use of org.apache.metron.indexing.dao.search.FieldType in project metron by apache.

the class ElasticsearchDao method buildGroupResponse.

/**
 * Build a group response.
 * @param groupRequest The original group request.
 * @param response The search response.
 * @return A group response.
 * @throws InvalidSearchException
 */
private GroupResponse buildGroupResponse(GroupRequest groupRequest, org.elasticsearch.action.search.SearchResponse response) throws InvalidSearchException {
    // build the search response
    Map<String, FieldType> commonColumnMetadata;
    try {
        commonColumnMetadata = getColumnMetadata(groupRequest.getIndices());
    } catch (IOException e) {
        throw new InvalidSearchException(String.format("Could not get common column metadata for indices %s", Arrays.toString(groupRequest.getIndices().toArray())));
    }
    GroupResponse groupResponse = new GroupResponse();
    groupResponse.setGroupedBy(groupRequest.getGroups().get(0).getField());
    groupResponse.setGroupResults(getGroupResults(groupRequest, 0, response.getAggregations(), commonColumnMetadata));
    return groupResponse;
}