Search in sources :

Example 76 with SearchSourceBuilder

use of org.elasticsearch.search.builder.SearchSourceBuilder in project engine by craftercms.

the class ContentTypeBasedDataFetcher method doGet.

/**
 * {@inheritDoc}
 */
@Override
public Object doGet(final DataFetchingEnvironment env) {
    Field field = env.getMergedField().getSingleField();
    String fieldName = field.getName();
    // Get arguments for pagination & sorting
    int offset = Optional.ofNullable(env.<Integer>getArgument(ARG_NAME_OFFSET)).orElse(0);
    int limit = Optional.ofNullable(env.<Integer>getArgument(ARG_NAME_LIMIT)).orElse(defaultLimit);
    String sortBy = Optional.ofNullable(env.<String>getArgument(ARG_NAME_SORT_BY)).orElse(defaultSortField);
    String sortOrder = Optional.ofNullable(env.<String>getArgument(ARG_NAME_SORT_ORDER)).orElse(defaultSortOrder);
    List<String> queryFieldIncludes = new LinkedList<>();
    // Add content-type to includes, we might need it for a GraphQL TypeResolver
    queryFieldIncludes.add(QUERY_FIELD_NAME_CONTENT_TYPE);
    List<Map<String, Object>> items = new LinkedList<>();
    Map<String, Object> result = new HashMap<>(2);
    result.put(FIELD_NAME_ITEMS, items);
    // Setup the ES query
    SearchSourceBuilder source = new SearchSourceBuilder();
    BoolQueryBuilder query = boolQuery();
    source.query(query).from(offset).size(limit).sort(sortBy, SortOrder.fromString(sortOrder));
    StopWatch watch = new StopWatch(field.getName() + " - " + field.getAlias());
    watch.start("build filters");
    // Filter by the content-type
    switch(fieldName) {
        case FIELD_NAME_CONTENT_ITEMS:
            query.filter(existsQuery(QUERY_FIELD_NAME_CONTENT_TYPE));
            break;
        case FIELD_NAME_PAGES:
            query.filter(regexpQuery(QUERY_FIELD_NAME_CONTENT_TYPE, CONTENT_TYPE_REGEX_PAGE));
            break;
        case FIELD_NAME_COMPONENTS:
            query.filter(regexpQuery(QUERY_FIELD_NAME_CONTENT_TYPE, CONTENT_TYPE_REGEX_COMPONENT));
            break;
        default:
            // Get the content-type name from the field name
            query.filter(termQuery(QUERY_FIELD_NAME_CONTENT_TYPE, getOriginalName(fieldName)));
            break;
    }
    // Check the selected fields to build the ES query
    Optional<Field> itemsField = field.getSelectionSet().getSelections().stream().map(f -> (Field) f).filter(f -> f.getName().equals(FIELD_NAME_ITEMS)).findFirst();
    if (itemsField.isPresent()) {
        List<Selection> selections = itemsField.get().getSelectionSet().getSelections();
        selections.forEach(selection -> processSelection(StringUtils.EMPTY, selection, query, queryFieldIncludes, env));
    }
    // Only fetch the selected fields for better performance
    source.fetchSource(queryFieldIncludes.toArray(new String[0]), new String[0]);
    watch.stop();
    logger.debug("Executing query: {}", source);
    watch.start("searching items");
    SearchResponse response = elasticsearch.search(new SearchRequest().source(source));
    watch.stop();
    watch.start("processing items");
    result.put(FIELD_NAME_TOTAL, response.getHits().totalHits);
    if (response.getHits().totalHits > 0) {
        for (SearchHit hit : response.getHits().getHits()) {
            items.add(fixItems(hit.getSourceAsMap()));
        }
    }
    watch.stop();
    if (logger.isTraceEnabled()) {
        logger.trace(watch.prettyPrint());
    }
    return result;
}
Also used : ObjectValue(graphql.language.ObjectValue) DataFetchingEnvironment(graphql.schema.DataFetchingEnvironment) FloatValue(graphql.language.FloatValue) Value(graphql.language.Value) LoggerFactory(org.slf4j.LoggerFactory) FragmentSpread(graphql.language.FragmentSpread) HashMap(java.util.HashMap) SearchRequest(org.elasticsearch.action.search.SearchRequest) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) StringUtils(org.apache.commons.lang3.StringUtils) ElasticsearchWrapper(org.craftercms.search.elasticsearch.ElasticsearchWrapper) LinkedHashMap(java.util.LinkedHashMap) Selection(graphql.language.Selection) VariableReference(graphql.language.VariableReference) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) DataFetcher(graphql.schema.DataFetcher) LinkedList(java.util.LinkedList) SearchHit(org.elasticsearch.search.SearchHit) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Logger(org.slf4j.Logger) MapUtils(org.apache.commons.collections.MapUtils) ObjectField(graphql.language.ObjectField) StopWatch(org.springframework.util.StopWatch) Field(graphql.language.Field) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) Argument(graphql.language.Argument) List(java.util.List) StringValue(graphql.language.StringValue) ArrayValue(graphql.language.ArrayValue) IntValue(graphql.language.IntValue) SortOrder(org.elasticsearch.search.sort.SortOrder) Optional(java.util.Optional) FragmentDefinition(graphql.language.FragmentDefinition) Required(org.springframework.beans.factory.annotation.Required) InlineFragment(graphql.language.InlineFragment) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) Collections(java.util.Collections) SchemaUtils(org.craftercms.engine.graphql.SchemaUtils) BooleanValue(graphql.language.BooleanValue) SearchRequest(org.elasticsearch.action.search.SearchRequest) SearchHit(org.elasticsearch.search.SearchHit) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Selection(graphql.language.Selection) LinkedList(java.util.LinkedList) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) StopWatch(org.springframework.util.StopWatch) SearchResponse(org.elasticsearch.action.search.SearchResponse) ObjectField(graphql.language.ObjectField) Field(graphql.language.Field) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map)

Example 77 with SearchSourceBuilder

use of org.elasticsearch.search.builder.SearchSourceBuilder in project sonarqube by SonarSource.

the class ProjectMeasuresIndexerTest method assertThatProjectHasTag.

private void assertThatProjectHasTag(ComponentDto project, String expectedTag) {
    SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()).source(new SearchSourceBuilder().query(boolQuery().filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())).filter(termQuery(FIELD_TAGS, expectedTag))));
    assertThat(es.client().search(request).getHits().getHits()).extracting(SearchHit::getId).contains(project.uuid());
}
Also used : SearchRequest(org.elasticsearch.action.search.SearchRequest) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Example 78 with SearchSourceBuilder

use of org.elasticsearch.search.builder.SearchSourceBuilder in project sonarqube by SonarSource.

the class ProjectMeasuresIndexerTest method assertThatQualifierIs.

private void assertThatQualifierIs(String qualifier, String... componentsUuid) {
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(boolQuery().filter(termQuery(FIELD_INDEX_TYPE, TYPE_PROJECT_MEASURES.getName())).filter(termQuery(FIELD_QUALIFIER, qualifier)).filter(termsQuery(FIELD_UUID, componentsUuid)));
    SearchRequest request = prepareSearch(TYPE_PROJECT_MEASURES.getMainType()).source(searchSourceBuilder);
    assertThat(es.client().search(request).getHits().getHits()).extracting(SearchHit::getId).containsExactlyInAnyOrder(componentsUuid);
}
Also used : SearchRequest(org.elasticsearch.action.search.SearchRequest) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Example 79 with SearchSourceBuilder

use of org.elasticsearch.search.builder.SearchSourceBuilder in project sonarqube by SonarSource.

the class RuleIndex method listTags.

public List<String> listTags(@Nullable String query, int size) {
    int maxPageSize = 500;
    checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize);
    if (size <= 0) {
        return emptyList();
    }
    TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME_FOR_TAGS).field(FIELD_RULE_TAGS).size(size).order(BucketOrder.key(true)).minDocCount(1);
    ofNullable(query).map(EsUtils::escapeSpecialRegexChars).map(queryString -> ".*" + queryString + ".*").map(s -> new IncludeExclude(s, null)).ifPresent(termsAggregation::includeExclude);
    SearchRequest request = EsClient.prepareSearch(TYPE_RULE.getMainType()).source(new SearchSourceBuilder().query(matchAllQuery()).size(0).aggregation(termsAggregation));
    SearchResponse esResponse = client.search(request);
    return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME_FOR_TAGS));
}
Also used : JavaTokenizer(org.sonar.server.es.textsearch.JavaTokenizer) Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) SortBuilders(org.elasticsearch.search.sort.SortBuilders) SCROLL_TIME_IN_MINUTES(org.sonar.server.es.EsUtils.SCROLL_TIME_IN_MINUTES) FIELD_RULE_INTERNAL_KEY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_INTERNAL_KEY) FIELD_RULE_CREATED_AT(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_CREATED_AT) TYPE_RULE(org.sonar.server.rule.index.RuleIndexDefinition.TYPE_RULE) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) FIELD_RULE_TAGS(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TAGS) StickyFacetBuilder(org.sonar.server.es.StickyFacetBuilder) EsClient(org.sonar.server.es.EsClient) FACET_DEFAULT_SIZE(org.sonar.server.es.StickyFacetBuilder.FACET_DEFAULT_SIZE) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SEARCH_WORDS_ANALYZER(org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_WORDS_ANALYZER) Map(java.util.Map) FIELD_RULE_STATUS(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_STATUS) SearchResponse(org.elasticsearch.action.search.SearchResponse) BucketOrder(org.elasticsearch.search.aggregations.BucketOrder) FIELD_RULE_UPDATED_AT(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_UPDATED_AT) FIELD_ACTIVE_RULE_INHERITANCE(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_INHERITANCE) JoinAggregationBuilders(org.elasticsearch.join.aggregations.JoinAggregationBuilders) FIELD_RULE_OWASP_TOP_10(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_OWASP_TOP_10) TimeValue(org.elasticsearch.core.TimeValue) FIELD_RULE_NAME(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_NAME) QueryBuilders.matchPhraseQuery(org.elasticsearch.index.query.QueryBuilders.matchPhraseQuery) QueryBuilders.boolQuery(org.elasticsearch.index.query.QueryBuilders.boolQuery) System2(org.sonar.api.utils.System2) SearchIdResult(org.sonar.server.es.SearchIdResult) Collections.emptyList(java.util.Collections.emptyList) VULNERABILITY(org.sonar.api.rules.RuleType.VULNERABILITY) Collection(java.util.Collection) FIELD_RULE_IS_EXTERNAL(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_IS_EXTERNAL) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) SORTABLE_ANALYZER(org.sonar.server.es.newindex.DefaultIndexSettingsElement.SORTABLE_ANALYZER) ScoreMode(org.apache.lucene.search.join.ScoreMode) FIELD_RULE_REPOSITORY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_REPOSITORY) List(java.util.List) SearchOptions(org.sonar.server.es.SearchOptions) QueryBuilders.matchQuery(org.elasticsearch.index.query.QueryBuilders.matchQuery) HasParentQueryBuilder(org.elasticsearch.join.query.HasParentQueryBuilder) FIELD_RULE_IS_TEMPLATE(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_IS_TEMPLATE) SortOrder(org.elasticsearch.search.sort.SortOrder) QProfileDto(org.sonar.db.qualityprofile.QProfileDto) FIELD_RULE_KEY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_KEY) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) TRUE(java.lang.Boolean.TRUE) Joiner(com.google.common.base.Joiner) IncludeExclude(org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude) FIELD_RULE_SANS_TOP_25(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_SANS_TOP_25) JoinQueryBuilders(org.elasticsearch.join.query.JoinQueryBuilders) AggregationBuilder(org.elasticsearch.search.aggregations.AggregationBuilder) RuleStatus(org.sonar.api.rule.RuleStatus) HashMap(java.util.HashMap) SearchRequest(org.elasticsearch.action.search.SearchRequest) Function(java.util.function.Function) FIELD_RULE_LANGUAGE(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_LANGUAGE) EsUtils(org.sonar.server.es.EsUtils) ArrayList(java.util.ArrayList) RuleType(org.sonar.api.rules.RuleType) EsUtils.scrollIds(org.sonar.server.es.EsUtils.scrollIds) ENGLISH_HTML_ANALYZER(org.sonar.server.es.newindex.DefaultIndexSettingsElement.ENGLISH_HTML_ANALYZER) SEARCH_GRAMS_ANALYZER(org.sonar.server.es.newindex.DefaultIndexSettingsElement.SEARCH_GRAMS_ANALYZER) Operator(org.elasticsearch.index.query.Operator) Severity(org.sonar.api.rule.Severity) FIELD_ACTIVE_RULE_SEVERITY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_SEVERITY) FIELD_RULE_CWE(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_CWE) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) SECURITY_HOTSPOT(org.sonar.api.rules.RuleType.SECURITY_HOTSPOT) FIELD_ACTIVE_RULE_PROFILE_UUID(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_ACTIVE_RULE_PROFILE_UUID) FIELD_INDEX_TYPE(org.sonar.server.es.IndexType.FIELD_INDEX_TYPE) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) MoreCollectors(org.sonar.core.util.stream.MoreCollectors) Nullable(javax.annotation.Nullable) QueryBuilders.termsQuery(org.elasticsearch.index.query.QueryBuilders.termsQuery) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) FALSE(java.lang.Boolean.FALSE) FIELD_RULE_SEVERITY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_SEVERITY) FIELD_RULE_SONARSOURCE_SECURITY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_SONARSOURCE_SECURITY) QueryBuilders.matchAllQuery(org.elasticsearch.index.query.QueryBuilders.matchAllQuery) Iterator(java.util.Iterator) Optional.ofNullable(java.util.Optional.ofNullable) FIELD_RULE_HTML_DESCRIPTION(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_HTML_DESCRIPTION) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) FIELD_RULE_RULE_KEY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_RULE_KEY) FIELD_RULE_TEMPLATE_KEY(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TEMPLATE_KEY) FIELD_RULE_TYPE(org.sonar.server.rule.index.RuleIndexDefinition.FIELD_RULE_TYPE) EsUtils.optimizeScrollRequest(org.sonar.server.es.EsUtils.optimizeScrollRequest) TYPE_ACTIVE_RULE(org.sonar.server.rule.index.RuleIndexDefinition.TYPE_ACTIVE_RULE) DefaultIndexSettings(org.sonar.server.es.newindex.DefaultIndexSettings) SearchRequest(org.elasticsearch.action.search.SearchRequest) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) IncludeExclude(org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) SearchResponse(org.elasticsearch.action.search.SearchResponse)

Example 80 with SearchSourceBuilder

use of org.elasticsearch.search.builder.SearchSourceBuilder in project sonarqube by SonarSource.

the class BulkIndexerTest method bulk_delete.

@Test
public void bulk_delete() {
    int max = 500;
    int removeFrom = 200;
    FakeDoc[] docs = new FakeDoc[max];
    for (int i = 0; i < max; i++) {
        docs[i] = FakeIndexDefinition.newDoc(i);
    }
    es.putDocuments(TYPE_FAKE, docs);
    assertThat(count()).isEqualTo(max);
    SearchRequest req = EsClient.prepareSearch(TYPE_FAKE).source(new SearchSourceBuilder().query(QueryBuilders.rangeQuery(FakeIndexDefinition.INT_FIELD).gte(removeFrom)));
    BulkIndexer.delete(es.client(), TYPE_FAKE, req);
    assertThat(count()).isEqualTo(removeFrom);
}
Also used : SearchRequest(org.elasticsearch.action.search.SearchRequest) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Test(org.junit.Test)

Aggregations

SearchSourceBuilder (org.elasticsearch.search.builder.SearchSourceBuilder)152 SearchResponse (org.elasticsearch.action.search.SearchResponse)77 SearchRequest (org.elasticsearch.action.search.SearchRequest)52 ElasticsearchAssertions.assertSearchResponse (org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse)37 Matchers.containsString (org.hamcrest.Matchers.containsString)32 IOException (java.io.IOException)31 QueryBuilder (org.elasticsearch.index.query.QueryBuilder)28 BoolQueryBuilder (org.elasticsearch.index.query.BoolQueryBuilder)26 List (java.util.List)25 SearchSourceBuilder (org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder)24 SearchSourceBuilder (org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder)24 Search (io.searchbox.core.Search)23 Map (java.util.Map)23 Set (java.util.Set)19 ArrayList (java.util.ArrayList)18 Collectors (java.util.stream.Collectors)18 Test (org.junit.Test)18 Collections (java.util.Collections)15 SearchHit (org.elasticsearch.search.SearchHit)15 TermsAggregationBuilder (org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder)15