Search in sources :

Example 96 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project sonarqube by SonarSource.

the class ProjectMeasuresIndex method addFacets.

private static void addFacets(SearchSourceBuilder esRequest, SearchOptions options, RequestFiltersComputer filtersComputer, ProjectMeasuresQuery query) {
    TopAggregationHelper topAggregationHelper = new TopAggregationHelper(filtersComputer, new SubAggregationHelper());
    options.getFacets().stream().map(FACETS_BY_NAME::get).filter(Objects::nonNull).map(facet -> facet.getFacetBuilder().buildFacet(facet, query, topAggregationHelper)).forEach(esRequest::aggregation);
}
Also used : ArrayListMultimap(com.google.common.collect.ArrayListMultimap) Arrays(java.util.Arrays) SECURITY_HOTSPOTS_REVIEWED_KEY(org.sonar.api.measures.CoreMetrics.SECURITY_HOTSPOTS_REVIEWED_KEY) RangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder) EsClient(org.sonar.server.es.EsClient) DESC(org.elasticsearch.search.sort.SortOrder.DESC) SimpleFieldFilterScope(org.sonar.server.es.searchrequest.TopAggregationDefinition.SimpleFieldFilterScope) SimpleFieldTopAggregationDefinition(org.sonar.server.es.searchrequest.SimpleFieldTopAggregationDefinition) Bucket(org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) Map(java.util.Map) BucketOrder(org.elasticsearch.search.aggregations.BucketOrder) FIELD_LANGUAGES(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_LANGUAGES) FILTER_LANGUAGES(org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_LANGUAGES) RELIABILITY_RATING_KEY(org.sonar.api.measures.CoreMetrics.RELIABILITY_RATING_KEY) NEW_COVERAGE_KEY(org.sonar.api.measures.CoreMetrics.NEW_COVERAGE_KEY) FIELD_QUALITY_GATE_STATUS(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALITY_GATE_STATUS) SUB_FIELD_MEASURES_KEY(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.SUB_FIELD_MEASURES_KEY) FIELD_MEASURES_MEASURE_VALUE(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_MEASURE_VALUE) QueryBuilders.boolQuery(org.elasticsearch.index.query.QueryBuilders.boolQuery) FIELD_NCLOC_DISTRIBUTION_LANGUAGE(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_LANGUAGE) SORT_BY_NAME(org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_NAME) Set(java.util.Set) NEW_DUPLICATED_LINES_DENSITY_KEY(org.sonar.api.measures.CoreMetrics.NEW_DUPLICATED_LINES_DENSITY_KEY) FIELD_MEASURES_MEASURE_KEY(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES_MEASURE_KEY) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) ScoreMode(org.apache.lucene.search.join.ScoreMode) Sum(org.elasticsearch.search.aggregations.metrics.Sum) MAX_PAGE_SIZE(org.sonarqube.ws.client.project.ProjectsWsParameters.MAX_PAGE_SIZE) NEW_RELIABILITY_RATING_KEY(org.sonar.api.measures.CoreMetrics.NEW_RELIABILITY_RATING_KEY) ALERT_STATUS_KEY(org.sonar.api.measures.CoreMetrics.ALERT_STATUS_KEY) Stream(java.util.stream.Stream) NestedFieldTopAggregationDefinition(org.sonar.server.es.searchrequest.NestedFieldTopAggregationDefinition) QueryBuilders.rangeQuery(org.elasticsearch.index.query.QueryBuilders.rangeQuery) EsUtils.termsToMap(org.sonar.server.es.EsUtils.termsToMap) AggregationBuilders.sum(org.elasticsearch.search.aggregations.AggregationBuilders.sum) AggregationBuilders.filters(org.elasticsearch.search.aggregations.AggregationBuilders.filters) ALERT_STATUS(org.sonar.server.measure.index.ProjectMeasuresIndex.Facet.ALERT_STATUS) WebAuthorizationTypeSupport(org.sonar.server.permission.index.WebAuthorizationTypeSupport) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) STICKY(org.sonar.server.es.searchrequest.TopAggregationDefinition.STICKY) FIELD_INDEX_TYPE(org.sonar.server.es.IndexType.FIELD_INDEX_TYPE) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) Nullable(javax.annotation.Nullable) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) SECURITY_RATING_KEY(org.sonar.api.measures.CoreMetrics.SECURITY_RATING_KEY) NEW_LINES_KEY(org.sonar.api.measures.CoreMetrics.NEW_LINES_KEY) FIELD_KEY(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_KEY) RequestFiltersComputer(org.sonar.server.es.searchrequest.RequestFiltersComputer) FIELD_NAME(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NAME) MetricCriterion(org.sonar.server.measure.index.ProjectMeasuresQuery.MetricCriterion) QUALITY_GATE_STATUS(org.sonar.server.measure.index.ProjectMeasuresDoc.QUALITY_GATE_STATUS) TotalHits(org.apache.lucene.search.TotalHits) QueryBuilders.termQuery(org.elasticsearch.index.query.QueryBuilders.termQuery) COVERAGE_KEY(org.sonar.api.measures.CoreMetrics.COVERAGE_KEY) Qualifiers(org.sonar.api.resources.Qualifiers) NCLOC_KEY(org.sonar.api.measures.CoreMetrics.NCLOC_KEY) FILTER_TAGS(org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_TAGS) KeyedFilter(org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter) NestedFieldFilterScope(org.sonar.server.es.searchrequest.TopAggregationDefinition.NestedFieldFilterScope) QueryBuilders.nestedQuery(org.elasticsearch.index.query.QueryBuilders.nestedQuery) NEW_SECURITY_RATING_KEY(org.sonar.api.measures.CoreMetrics.NEW_SECURITY_RATING_KEY) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SORT_BY_LAST_ANALYSIS_DATE(org.sonar.server.measure.index.ProjectMeasuresQuery.SORT_BY_LAST_ANALYSIS_DATE) SearchResponse(org.elasticsearch.action.search.SearchResponse) NEW_MAINTAINABILITY_RATING_KEY(org.sonar.api.measures.CoreMetrics.NEW_MAINTAINABILITY_RATING_KEY) TAGS(org.sonar.server.measure.index.ProjectMeasuresIndex.Facet.TAGS) FIELD_ANALYSED_AT(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_ANALYSED_AT) DefaultIndexSettingsElement(org.sonar.server.es.newindex.DefaultIndexSettingsElement) SubAggregationHelper(org.sonar.server.es.searchrequest.SubAggregationHelper) TYPE_PROJECT_MEASURES(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.TYPE_PROJECT_MEASURES) System2(org.sonar.api.utils.System2) SearchIdResult(org.sonar.server.es.SearchIdResult) Terms(org.elasticsearch.search.aggregations.bucket.terms.Terms) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) NestedSortBuilder(org.elasticsearch.search.sort.NestedSortBuilder) TopAggregationDefinition(org.sonar.server.es.searchrequest.TopAggregationDefinition) FilterAggregationBuilder(org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder) FILTER_QUALIFIER(org.sonarqube.ws.client.project.ProjectsWsParameters.FILTER_QUALIFIER) Objects(java.util.Objects) TopAggregationHelper(org.sonar.server.es.searchrequest.TopAggregationHelper) List(java.util.List) SearchOptions(org.sonar.server.es.SearchOptions) SQALE_RATING_KEY(org.sonar.api.measures.CoreMetrics.SQALE_RATING_KEY) ServerSide(org.sonar.api.server.ServerSide) AllFilters(org.sonar.server.es.searchrequest.RequestFiltersComputer.AllFilters) MoreCollectors.uniqueIndex(org.sonar.core.util.stream.MoreCollectors.uniqueIndex) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) IntStream(java.util.stream.IntStream) IncludeExclude(org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude) FIELD_NCLOC_DISTRIBUTION(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION) AbstractAggregationBuilder(org.elasticsearch.search.aggregations.AbstractAggregationBuilder) EsUtils.escapeSpecialRegexChars(org.sonar.server.es.EsUtils.escapeSpecialRegexChars) Multimap(com.google.common.collect.Multimap) FIELD_NCLOC_DISTRIBUTION_NCLOC(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_NCLOC_DISTRIBUTION_NCLOC) Function(java.util.function.Function) NEW_SECURITY_REVIEW_RATING_KEY(org.sonar.api.measures.CoreMetrics.NEW_SECURITY_REVIEW_RATING_KEY) MoreCollectors.toSet(org.sonar.core.util.stream.MoreCollectors.toSet) Nested(org.elasticsearch.search.aggregations.bucket.nested.Nested) SECURITY_REVIEW_RATING_KEY(org.sonar.api.measures.CoreMetrics.SECURITY_REVIEW_RATING_KEY) MoreCollectors(org.sonar.core.util.stream.MoreCollectors) QueryBuilders.termsQuery(org.elasticsearch.index.query.QueryBuilders.termsQuery) Optional.ofNullable(java.util.Optional.ofNullable) DUPLICATED_LINES_DENSITY_KEY(org.sonar.api.measures.CoreMetrics.DUPLICATED_LINES_DENSITY_KEY) FIELD_QUALIFIER(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_QUALIFIER) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) LANGUAGES(org.sonar.server.measure.index.ProjectMeasuresIndex.Facet.LANGUAGES) Consumer(java.util.function.Consumer) FIELD_MEASURES(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_MEASURES) ASC(org.elasticsearch.search.sort.SortOrder.ASC) NO_EXTRA_FILTER(org.sonar.server.es.searchrequest.TopAggregationHelper.NO_EXTRA_FILTER) FIELD_TAGS(org.sonar.server.measure.index.ProjectMeasuresIndexDefinition.FIELD_TAGS) NEW_SECURITY_HOTSPOTS_REVIEWED_KEY(org.sonar.api.measures.CoreMetrics.NEW_SECURITY_HOTSPOTS_REVIEWED_KEY) Metric(org.sonar.api.measures.Metric) SubAggregationHelper(org.sonar.server.es.searchrequest.SubAggregationHelper) Objects(java.util.Objects) TopAggregationHelper(org.sonar.server.es.searchrequest.TopAggregationHelper)

Example 97 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project sonarqube by SonarSource.

the class CreateActionTest method create_user.

@Test
public void create_user() {
    logInAsSystemAdministrator();
    CreateWsResponse response = call(CreateRequest.builder().setLogin("john").setName("John").setEmail("john@email.com").setScmAccounts(singletonList("jn")).setPassword("1234").build());
    assertThat(response.getUser()).extracting(User::getLogin, User::getName, User::getEmail, User::getScmAccountsList, User::getLocal).containsOnly("john", "John", "john@email.com", singletonList("jn"), true);
    // exists in index
    assertThat(es.client().search(EsClient.prepareSearch(UserIndexDefinition.TYPE_USER).source(new SearchSourceBuilder().query(boolQuery().must(termQuery(FIELD_LOGIN, "john")).must(termQuery(FIELD_NAME, "John")).must(termQuery(FIELD_EMAIL, "john@email.com")).must(termQuery(FIELD_SCM_ACCOUNTS, "jn"))))).getHits().getHits()).hasSize(1);
    // exists in db
    Optional<UserDto> dbUser = db.users().selectUserByLogin("john");
    assertThat(dbUser).isPresent();
    assertThat(dbUser.get().isRoot()).isFalse();
    // member of default group
    assertThat(db.users().selectGroupUuidsOfUser(dbUser.get())).containsOnly(defaultGroup.getUuid());
}
Also used : UserDto(org.sonar.db.user.UserDto) UserTesting.newUserDto(org.sonar.db.user.UserTesting.newUserDto) CreateWsResponse(org.sonarqube.ws.Users.CreateWsResponse) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Test(org.junit.Test)

Example 98 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project metron by apache.

the class ElasticsearchRetrieveLatestDao method searchByGuids.

/**
 * Return the search hit based on the UUID and sensor type.
 * A callback can be specified to transform the hit into a type T.
 * If more than one hit happens, the first one will be returned.
 */
<T> List<T> searchByGuids(Collection<String> guids, Collection<String> sensorTypes, Function<SearchHit, Optional<T>> callback) throws IOException {
    if (guids == null || guids.isEmpty()) {
        return Collections.emptyList();
    }
    // should match any of the guids
    // the 'guid' field must be of type 'keyword' or this term query will not match
    BoolQueryBuilder guidQuery = boolQuery().must(termsQuery(Constants.GUID, guids));
    // should match any of the sensor types
    BoolQueryBuilder sensorQuery = boolQuery();
    sensorTypes.forEach(sensorType -> sensorQuery.should(typeQuery(sensorType + "_doc")));
    // must have a match for both guid and sensor
    BoolQueryBuilder query = boolQuery().must(guidQuery).must(sensorQuery);
    // submit the search
    SearchResponse response;
    try {
        SearchSourceBuilder source = new SearchSourceBuilder().query(query).size(guids.size());
        SearchRequest request = new SearchRequest().source(source);
        response = submitter.submitSearch(request);
    } catch (InvalidSearchException e) {
        throw new IOException(e);
    }
    // transform the search hits to results using the callback
    List<T> results = new ArrayList<>();
    for (SearchHit hit : response.getHits()) {
        Optional<T> result = callback.apply(hit);
        result.ifPresent(r -> results.add(r));
    }
    return results;
}
Also used : SearchRequest(org.elasticsearch.action.search.SearchRequest) InvalidSearchException(org.apache.metron.indexing.dao.search.InvalidSearchException) SearchHit(org.elasticsearch.search.SearchHit) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) ArrayList(java.util.ArrayList) IOException(java.io.IOException) SearchResponse(org.elasticsearch.action.search.SearchResponse) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Example 99 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project metron by apache.

the class ElasticsearchUtils method queryAllResults.

/**
 * Elasticsearch queries default to 10 records returned.  Some internal queries require that all
 * results are returned.  Rather than setting an arbitrarily high size, this method pages through results
 * and returns them all in a single SearchResponse.
 * @param qb A QueryBuilder that provides the query to be run.
 * @return A SearchResponse containing the appropriate results.
 */
public static SearchResponse queryAllResults(RestHighLevelClient transportClient, QueryBuilder qb, String index, int pageSize) throws IOException {
    org.elasticsearch.action.search.SearchRequest request = new org.elasticsearch.action.search.SearchRequest();
    SearchSourceBuilder builder = new SearchSourceBuilder();
    builder.query(qb);
    builder.size(pageSize);
    builder.fetchSource(true);
    builder.storedField("*");
    request.source(builder);
    request.indices(index);
    org.elasticsearch.action.search.SearchResponse esResponse = transportClient.search(request);
    List<SearchResult> allResults = getSearchResults(esResponse);
    long total = esResponse.getHits().getTotalHits();
    if (total > pageSize) {
        int pages = (int) (total / pageSize) + 1;
        for (int i = 1; i < pages; i++) {
            int from = i * pageSize;
            builder.from(from);
            esResponse = transportClient.search(request);
            allResults.addAll(getSearchResults(esResponse));
        }
    }
    SearchResponse searchResponse = new SearchResponse();
    searchResponse.setTotal(total);
    searchResponse.setResults(allResults);
    return searchResponse;
}
Also used : SearchResult(org.apache.metron.indexing.dao.search.SearchResult) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) SearchResponse(org.apache.metron.indexing.dao.search.SearchResponse)

Example 100 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project dq-easy-cloud by dq-open-cloud.

the class TransportClient method count.

/**
 * Count文档
 *
 * @throws Exception
 */
@Test
public void count() throws Exception {
    String[] name = new String[] { "T:o\"m-", "Jerry" };
    String from = "2016-09-01T00:00:00";
    String to = "2016-10-01T00:00:00";
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery("name", name)).must(QueryBuilders.rangeQuery("birth").gte(from).lte(to));
    searchSourceBuilder.query(queryBuilder);
    String query = searchSourceBuilder.toString();
    System.out.println(query);
    Count count = new Count.Builder().addIndex(indexName).addType(typeName).query(query).build();
    CountResult results = jestClient.execute(count);
    Double counts = results.getCount();
    System.out.println("Count:" + counts);
}
Also used : GsonBuilder(com.google.gson.GsonBuilder) HighlightBuilder(org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Count(io.searchbox.core.Count) CountResult(io.searchbox.core.CountResult) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Test(org.junit.Test)

Aggregations

SearchSourceBuilder (org.elasticsearch.search.builder.SearchSourceBuilder)193 SearchResponse (org.elasticsearch.action.search.SearchResponse)101 SearchRequest (org.elasticsearch.action.search.SearchRequest)76 QueryBuilder (org.elasticsearch.index.query.QueryBuilder)41 ElasticsearchAssertions.assertSearchResponse (org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse)37 IOException (java.io.IOException)36 BoolQueryBuilder (org.elasticsearch.index.query.BoolQueryBuilder)34 Search (io.searchbox.core.Search)32 Matchers.containsString (org.hamcrest.Matchers.containsString)32 Test (org.junit.Test)29 List (java.util.List)26 Map (java.util.Map)25 SearchSourceBuilder (org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder)24 SearchSourceBuilder (org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder)24 ArrayList (java.util.ArrayList)20 TermsAggregationBuilder (org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder)20 Set (java.util.Set)19 Collectors (java.util.stream.Collectors)18 SearchHit (org.elasticsearch.search.SearchHit)18 FieldSortBuilder (org.elasticsearch.search.sort.FieldSortBuilder)16