Search in sources :

Example 41 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project opennms by OpenNMS.

the class AlarmElasticsearch5IT method getNumberOfAlarmsInEsWithUei.

private static int getNumberOfAlarmsInEsWithUei(InetSocketAddress esHttpAddr, String uei) throws IOException {
    JestClient client = null;
    try {
        JestClientFactory factory = new JestClientFactory();
        factory.setHttpClientConfig(new HttpClientConfig.Builder(String.format("http://%s:%d", esHttpAddr.getHostString(), esHttpAddr.getPort())).multiThreaded(true).build());
        client = factory.getObject();
        SearchResult response = client.execute(new Search.Builder(new SearchSourceBuilder().query(QueryBuilders.matchQuery("eventuei", EventConstants.IMPORT_FAILED_UEI)).toString()).addIndex("opennms-alarms*").build());
        LOG.debug("SEARCH RESPONSE: {}", response.toString());
        return response.getTotal();
    } finally {
        if (client != null) {
            client.shutdownClient();
        }
    }
}
Also used : HttpClientConfig(io.searchbox.client.config.HttpClientConfig) Search(io.searchbox.core.Search) SearchResult(io.searchbox.core.SearchResult) JestClient(io.searchbox.client.JestClient) JestClientFactory(io.searchbox.client.JestClientFactory) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Example 42 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project opennms by OpenNMS.

the class AbstractSyslogTestCase method pollForElasticsearchEventsUsingJest.

protected static void pollForElasticsearchEventsUsingJest(Supplier<InetSocketAddress> esTransportAddr, int numMessages) {
    with().pollInterval(15, SECONDS).await().atMost(5, MINUTES).until(() -> {
        JestClient client = null;
        try {
            JestClientFactory factory = new JestClientFactory();
            factory.setHttpClientConfig(new HttpClientConfig.Builder(String.format("http://%s:%d", esTransportAddr.get().getHostString(), esTransportAddr.get().getPort())).multiThreaded(true).build());
            client = factory.getObject();
            SearchResult response = client.execute(new Search.Builder(new SearchSourceBuilder().query(QueryBuilders.matchQuery("eventuei", "uei.opennms.org/vendor/cisco/syslog/SEC-6-IPACCESSLOGP/aclDeniedIPTraffic")).toString()).addIndex("opennms*").build());
            LOG.debug("SEARCH RESPONSE: {}", response.toString());
            // Sometimes, the first warm-up message is successful so treat both message counts as valid
            assertTrue("ES search hits was not equal to " + numMessages + ": " + response.getTotal(), (numMessages == response.getTotal()));
        // assertEquals("Event UEI did not match", "uei.opennms.org/vendor/cisco/syslog/SEC-6-IPACCESSLOGP/aclDeniedIPTraffic", response.getHits().getAt(0).getSource().get("eventuei"));
        // assertEquals("Event IP address did not match", "4.2.2.2", response.getHits().getAt(0).getSource().get("ipaddr"));
        } catch (Throwable e) {
            LOG.warn(e.getMessage(), e);
            return false;
        } finally {
            if (client != null) {
                client.shutdownClient();
            }
        }
        return true;
    });
}
Also used : HttpClientConfig(io.searchbox.client.config.HttpClientConfig) Search(io.searchbox.core.Search) SearchResult(io.searchbox.core.SearchResult) JestClient(io.searchbox.client.JestClient) JestClientFactory(io.searchbox.client.JestClientFactory) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Example 43 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project metron by apache.

the class ElasticsearchDao method buildGroupRequest.

/**
 * Builds a group search request.
 * @param groupRequest The Metron group request.
 * @param queryBuilder The search query.
 * @return An Elasticsearch search request.
 */
private org.elasticsearch.action.search.SearchRequest buildGroupRequest(GroupRequest groupRequest, QueryBuilder queryBuilder) {
    // handle groups
    TermsAggregationBuilder groups = getGroupsTermBuilder(groupRequest, 0);
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder).aggregation(groups);
    // return the search request
    String[] indices = wildcardIndices(groupRequest.getIndices());
    return new org.elasticsearch.action.search.SearchRequest().indices(indices).source(searchSourceBuilder);
}
Also used : SearchRequest(org.apache.metron.indexing.dao.search.SearchRequest) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Example 44 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project metron by apache.

the class ElasticsearchDao method buildSearchRequest.

/**
 * Builds an Elasticsearch search request.
 * @param searchRequest The Metron search request.
 * @param queryBuilder
 * @return An Elasticsearch search request.
 */
private org.elasticsearch.action.search.SearchRequest buildSearchRequest(SearchRequest searchRequest, QueryBuilder queryBuilder) throws InvalidSearchException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Got search request; request={}", ElasticsearchUtils.toJSON(searchRequest).orElse("???"));
    }
    SearchSourceBuilder searchBuilder = new SearchSourceBuilder().size(searchRequest.getSize()).from(searchRequest.getFrom()).query(queryBuilder).trackScores(true);
    List<String> fields = searchRequest.getFields();
    // column metadata needed to understand the type of each sort field
    Map<String, FieldType> meta;
    try {
        meta = getColumnMetadata(searchRequest.getIndices());
    } catch (IOException e) {
        throw new InvalidSearchException("Unable to get column metadata", e);
    }
    // handle sort fields
    for (SortField sortField : searchRequest.getSort()) {
        // what type is the sort field?
        FieldType sortFieldType = meta.getOrDefault(sortField.getField(), FieldType.OTHER);
        // sort order - if ascending missing values sorted last. otherwise, missing values sorted first
        org.elasticsearch.search.sort.SortOrder sortOrder = getElasticsearchSortOrder(sortField.getSortOrder());
        String missingSortOrder;
        if (sortOrder == org.elasticsearch.search.sort.SortOrder.DESC) {
            missingSortOrder = SORT_MISSING_LAST;
        } else {
            missingSortOrder = SORT_MISSING_FIRST;
        }
        // sort by the field - missing fields always last
        FieldSortBuilder sortBy = new FieldSortBuilder(sortField.getField()).order(sortOrder).missing(missingSortOrder).unmappedType(sortFieldType.getFieldType());
        searchBuilder.sort(sortBy);
    }
    // handle search fields
    if (fields != null) {
        searchBuilder.fetchSource("*", null);
    } else {
        searchBuilder.fetchSource(true);
    }
    List<String> facetFields = searchRequest.getFacetFields();
    // handle facet fields
    if (facetFields != null) {
        // https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/_bucket_aggregations.html
        for (String field : facetFields) {
            String name = getFacetAggregationName(field);
            TermsAggregationBuilder terms = AggregationBuilders.terms(name).field(field);
            // new TermsBuilder(name).field(field);
            searchBuilder.aggregation(terms);
        }
    }
    // return the search request
    String[] indices = wildcardIndices(searchRequest.getIndices());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Built Elasticsearch request; indices={}, request={}", indices, searchBuilder.toString());
    }
    return new org.elasticsearch.action.search.SearchRequest().indices(indices).source(searchBuilder);
}
Also used : SearchRequest(org.apache.metron.indexing.dao.search.SearchRequest) SortField(org.apache.metron.indexing.dao.search.SortField) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) IOException(java.io.IOException) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) FieldType(org.apache.metron.indexing.dao.search.FieldType) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) InvalidSearchException(org.apache.metron.indexing.dao.search.InvalidSearchException)

Example 45 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project herd by FINRAOS.

the class BusinessObjectDefinitionIndexSearchDaoImpl method findAllBusinessObjectDefinitions.

@Override
public ElasticsearchResponseDto findAllBusinessObjectDefinitions(String indexName, String documentType, Set<String> facetFieldsList) {
    LOGGER.info("Elasticsearch get all business object definition documents from index, indexName={} and documentType={}.", indexName, documentType);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.fetchSource(new String[] { DATA_PROVIDER_NAME_SOURCE, DESCRIPTION_SOURCE, DISPLAY_NAME_SOURCE, NAME_SOURCE, NAMESPACE_CODE_SOURCE }, null);
    ElasticsearchResponseDto elasticsearchResponseDto = new ElasticsearchResponseDto();
    // Create a search request and set the scroll time and scroll size
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(indexName);
    searchRequestBuilder.setTypes(documentType).setSource(searchSourceBuilder).addSort(SortBuilders.fieldSort(BUSINESS_OBJECT_DEFINITION_SORT_FIELD).order(SortOrder.ASC)).addSort(SortBuilders.fieldSort(NAMESPACE_CODE_SORT_FIELD).order(SortOrder.ASC));
    // Add aggregation builder if facet fields are present
    addFacetFieldAggregations(facetFieldsList, elasticsearchResponseDto, searchRequestBuilder, indexName);
    elasticsearchResponseDto.setBusinessObjectDefinitionIndexSearchResponseDtos(scrollSearchResultsIntoBusinessObjectDefinitionDto(searchRequestBuilder, indexName));
    return elasticsearchResponseDto;
}
Also used : SearchRequestBuilder(org.elasticsearch.action.search.SearchRequestBuilder) ElasticsearchResponseDto(org.finra.herd.model.dto.ElasticsearchResponseDto) ElasticsearchClientImpl(org.finra.herd.dao.helper.ElasticsearchClientImpl) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Aggregations

SearchSourceBuilder (org.elasticsearch.search.builder.SearchSourceBuilder)193 SearchResponse (org.elasticsearch.action.search.SearchResponse)101 SearchRequest (org.elasticsearch.action.search.SearchRequest)76 QueryBuilder (org.elasticsearch.index.query.QueryBuilder)41 ElasticsearchAssertions.assertSearchResponse (org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse)37 IOException (java.io.IOException)36 BoolQueryBuilder (org.elasticsearch.index.query.BoolQueryBuilder)34 Search (io.searchbox.core.Search)32 Matchers.containsString (org.hamcrest.Matchers.containsString)32 Test (org.junit.Test)29 List (java.util.List)26 Map (java.util.Map)25 SearchSourceBuilder (org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder)24 SearchSourceBuilder (org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder)24 ArrayList (java.util.ArrayList)20 TermsAggregationBuilder (org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder)20 Set (java.util.Set)19 Collectors (java.util.stream.Collectors)18 SearchHit (org.elasticsearch.search.SearchHit)18 FieldSortBuilder (org.elasticsearch.search.sort.FieldSortBuilder)16