use of org.opensearch.search.fetch.subphase.highlight.HighlightBuilder in project OpenSearch by opensearch-project.
the class ChildQuerySearchIT method testHighlightersIgnoreParentChild.
public void testHighlightersIgnoreParentChild() throws IOException {
assertAcked(prepareCreate("test").addMapping("doc", jsonBuilder().startObject().startObject("properties").startObject("join_field").field("type", "join").startObject("relations").field("parent-type", "child-type").endObject().endObject().startObject("searchText").field("type", "text").field("term_vector", "with_positions_offsets").field("index_options", "offsets").endObject().endObject().endObject()));
createIndexRequest("test", "parent-type", "parent-id", null, "searchText", "quick brown fox").get();
createIndexRequest("test", "child-type", "child-id", "parent-id", "searchText", "quick brown fox").get();
refresh();
String[] highlightTypes = new String[] { "plain", "fvh", "unified" };
for (String highlightType : highlightTypes) {
logger.info("Testing with highlight type [{}]", highlightType);
SearchResponse searchResponse = client().prepareSearch("test").setQuery(new BoolQueryBuilder().must(new MatchQueryBuilder("searchText", "fox")).must(new HasChildQueryBuilder("child-type", new MatchAllQueryBuilder(), ScoreMode.None))).highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("searchText").highlighterType(highlightType))).get();
assertHitCount(searchResponse, 1);
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("parent-id"));
HighlightField highlightField = searchResponse.getHits().getAt(0).getHighlightFields().get("searchText");
assertThat(highlightField.getFragments()[0].string(), equalTo("quick brown <em>fox</em>"));
searchResponse = client().prepareSearch("test").setQuery(new BoolQueryBuilder().must(new MatchQueryBuilder("searchText", "fox")).must(new HasParentQueryBuilder("parent-type", new MatchAllQueryBuilder(), false))).highlighter(new HighlightBuilder().field(new HighlightBuilder.Field("searchText").highlighterType(highlightType))).get();
assertHitCount(searchResponse, 1);
assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("child-id"));
highlightField = searchResponse.getHits().getAt(0).getHighlightFields().get("searchText");
assertThat(highlightField.getFragments()[0].string(), equalTo("quick brown <em>fox</em>"));
}
}
use of org.opensearch.search.fetch.subphase.highlight.HighlightBuilder in project OpenSearch by opensearch-project.
the class InnerHitsIT method testSimpleParentChild.
public void testSimpleParentChild() throws Exception {
assertAcked(prepareCreate("articles").addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties").startObject("join_field").field("type", "join").startObject("relations").field("article", "comment").endObject().endObject().startObject("title").field("type", "text").endObject().startObject("message").field("type", "text").field("fielddata", true).endObject().endObject().endObject().endObject()));
List<IndexRequestBuilder> requests = new ArrayList<>();
requests.add(createIndexRequest("articles", "article", "p1", null, "title", "quick brown fox"));
requests.add(createIndexRequest("articles", "comment", "c1", "p1", "message", "fox eat quick"));
requests.add(createIndexRequest("articles", "comment", "c2", "p1", "message", "fox ate rabbit x y z"));
requests.add(createIndexRequest("articles", "comment", "c3", "p1", "message", "rabbit got away"));
requests.add(createIndexRequest("articles", "article", "p2", null, "title", "big gray elephant"));
requests.add(createIndexRequest("articles", "comment", "c4", "p2", "message", "elephant captured"));
requests.add(createIndexRequest("articles", "comment", "c5", "p2", "message", "mice squashed by elephant x"));
requests.add(createIndexRequest("articles", "comment", "c6", "p2", "message", "elephant scared by mice x y"));
indexRandom(true, requests);
SearchResponse response = client().prepareSearch("articles").setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(new InnerHitBuilder())).get();
assertNoFailures(response);
assertHitCount(response, 1);
assertSearchHit(response, 1, hasId("p1"));
assertThat(response.getHits().getAt(0).getShard(), notNullValue());
assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getTotalHits().value, equalTo(2L));
assertThat(innerHits.getAt(0).getId(), equalTo("c1"));
assertThat(innerHits.getAt(1).getId(), equalTo("c2"));
final boolean seqNoAndTerm = randomBoolean();
response = client().prepareSearch("articles").setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None).innerHit(new InnerHitBuilder().setSeqNoAndPrimaryTerm(seqNoAndTerm))).get();
assertNoFailures(response);
assertHitCount(response, 1);
assertSearchHit(response, 1, hasId("p2"));
assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getTotalHits().value, equalTo(3L));
assertThat(innerHits.getAt(0).getId(), equalTo("c4"));
assertThat(innerHits.getAt(1).getId(), equalTo("c5"));
assertThat(innerHits.getAt(2).getId(), equalTo("c6"));
if (seqNoAndTerm) {
assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L));
assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L));
assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L));
assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L));
assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L));
assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L));
} else {
assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
assertThat(innerHits.getAt(0).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
assertThat(innerHits.getAt(1).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
assertThat(innerHits.getAt(2).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
}
response = client().prepareSearch("articles").setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(new InnerHitBuilder().addFetchField("message").setHighlightBuilder(new HighlightBuilder().field("message")).setExplain(true).setSize(1).addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap())))).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(message:fox"));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
response = client().prepareSearch("articles").setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(new InnerHitBuilder().addDocValueField("message").setSize(1))).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
}
use of org.opensearch.search.fetch.subphase.highlight.HighlightBuilder in project OpenSearch by opensearch-project.
the class InnerHitsIT method testSimpleNested.
public void testSimpleNested() throws Exception {
assertAcked(prepareCreate("articles").addMapping("article", jsonBuilder().startObject().startObject("article").startObject("properties").startObject("comments").field("type", "nested").startObject("properties").startObject("message").field("type", "text").field("fielddata", true).endObject().endObject().endObject().startObject("title").field("type", "text").endObject().endObject().endObject().endObject()));
List<IndexRequestBuilder> requests = new ArrayList<>();
requests.add(client().prepareIndex("articles").setId("1").setSource(jsonBuilder().startObject().field("title", "quick brown fox").startArray("comments").startObject().field("message", "fox eat quick").endObject().startObject().field("message", "fox ate rabbit x y z").endObject().startObject().field("message", "rabbit got away").endObject().endArray().endObject()));
requests.add(client().prepareIndex("articles").setId("2").setSource(jsonBuilder().startObject().field("title", "big gray elephant").startArray("comments").startObject().field("message", "elephant captured").endObject().startObject().field("message", "mice squashed by elephant x").endObject().startObject().field("message", "elephant scared by mice x y").endObject().endArray().endObject()));
indexRandom(true, requests);
SearchResponse response = client().prepareSearch("articles").setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder("comment"))).get();
assertNoFailures(response);
assertHitCount(response, 1);
assertSearchHit(response, 1, hasId("1"));
assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
SearchHits innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getTotalHits().value, equalTo(2L));
assertThat(innerHits.getHits().length, equalTo(2));
assertThat(innerHits.getAt(0).getId(), equalTo("1"));
assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(innerHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
assertThat(innerHits.getAt(1).getId(), equalTo("1"));
assertThat(innerHits.getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(innerHits.getAt(1).getNestedIdentity().getOffset(), equalTo(1));
response = client().prepareSearch("articles").setQuery(nestedQuery("comments", matchQuery("comments.message", "elephant"), ScoreMode.Avg).innerHit(new InnerHitBuilder("comment"))).get();
assertNoFailures(response);
assertHitCount(response, 1);
assertSearchHit(response, 1, hasId("2"));
assertThat(response.getHits().getAt(0).getShard(), notNullValue());
assertThat(response.getHits().getAt(0).getInnerHits().size(), equalTo(1));
innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getTotalHits().value, equalTo(3L));
assertThat(innerHits.getHits().length, equalTo(3));
assertThat(innerHits.getAt(0).getId(), equalTo("2"));
assertThat(innerHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(innerHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
assertThat(innerHits.getAt(1).getId(), equalTo("2"));
assertThat(innerHits.getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(innerHits.getAt(1).getNestedIdentity().getOffset(), equalTo(1));
assertThat(innerHits.getAt(2).getId(), equalTo("2"));
assertThat(innerHits.getAt(2).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(innerHits.getAt(2).getNestedIdentity().getOffset(), equalTo(2));
response = client().prepareSearch("articles").setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message")).setExplain(true).addFetchField("comments.mes*").addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap())).setSize(1))).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
assertThat(innerHits.getTotalHits().value, equalTo(2L));
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getHighlightFields().get("comments.message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(comments.message:fox in"));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
response = client().prepareSearch("articles").setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder().addDocValueField("comments.mes*").setSize(1))).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
}
use of org.opensearch.search.fetch.subphase.highlight.HighlightBuilder in project OpenSearch by opensearch-project.
the class SearchService method parseSource.
private void parseSource(DefaultSearchContext context, SearchSourceBuilder source, boolean includeAggregations) {
// nothing to parse...
if (source == null) {
return;
}
SearchShardTarget shardTarget = context.shardTarget();
QueryShardContext queryShardContext = context.getQueryShardContext();
context.from(source.from());
context.size(source.size());
Map<String, InnerHitContextBuilder> innerHitBuilders = new HashMap<>();
if (source.query() != null) {
InnerHitContextBuilder.extractInnerHits(source.query(), innerHitBuilders);
context.parsedQuery(queryShardContext.toQuery(source.query()));
}
if (source.postFilter() != null) {
InnerHitContextBuilder.extractInnerHits(source.postFilter(), innerHitBuilders);
context.parsedPostFilter(queryShardContext.toQuery(source.postFilter()));
}
if (innerHitBuilders.size() > 0) {
for (Map.Entry<String, InnerHitContextBuilder> entry : innerHitBuilders.entrySet()) {
try {
entry.getValue().build(context, context.innerHits());
} catch (IOException e) {
throw new SearchException(shardTarget, "failed to build inner_hits", e);
}
}
}
if (source.sorts() != null) {
try {
Optional<SortAndFormats> optionalSort = SortBuilder.buildSort(source.sorts(), context.getQueryShardContext());
if (optionalSort.isPresent()) {
context.sort(optionalSort.get());
}
} catch (IOException e) {
throw new SearchException(shardTarget, "failed to create sort elements", e);
}
}
context.trackScores(source.trackScores());
if (source.trackTotalHitsUpTo() != null && source.trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_ACCURATE && context.scrollContext() != null) {
throw new SearchException(shardTarget, "disabling [track_total_hits] is not allowed in a scroll context");
}
if (source.trackTotalHitsUpTo() != null) {
context.trackTotalHitsUpTo(source.trackTotalHitsUpTo());
}
if (source.minScore() != null) {
context.minimumScore(source.minScore());
}
if (source.profile()) {
context.setProfilers(new Profilers(context.searcher()));
}
if (source.timeout() != null) {
context.timeout(source.timeout());
}
context.terminateAfter(source.terminateAfter());
if (source.aggregations() != null && includeAggregations) {
try {
AggregatorFactories factories = source.aggregations().build(queryShardContext, null);
context.aggregations(new SearchContextAggregations(factories, multiBucketConsumerService.create()));
} catch (IOException e) {
throw new AggregationInitializationException("Failed to create aggregators", e);
}
}
if (source.suggest() != null) {
try {
context.suggest(source.suggest().build(queryShardContext));
} catch (IOException e) {
throw new SearchException(shardTarget, "failed to create SuggestionSearchContext", e);
}
}
if (source.rescores() != null) {
try {
for (RescorerBuilder<?> rescore : source.rescores()) {
context.addRescore(rescore.buildContext(queryShardContext));
}
} catch (IOException e) {
throw new SearchException(shardTarget, "failed to create RescoreSearchContext", e);
}
}
if (source.explain() != null) {
context.explain(source.explain());
}
if (source.fetchSource() != null) {
context.fetchSourceContext(source.fetchSource());
}
if (source.docValueFields() != null) {
FetchDocValuesContext docValuesContext = FetchDocValuesContext.create(context.mapperService(), source.docValueFields());
context.docValuesContext(docValuesContext);
}
if (source.fetchFields() != null) {
FetchFieldsContext fetchFieldsContext = new FetchFieldsContext(source.fetchFields());
context.fetchFieldsContext(fetchFieldsContext);
}
if (source.highlighter() != null) {
HighlightBuilder highlightBuilder = source.highlighter();
try {
context.highlight(highlightBuilder.build(queryShardContext));
} catch (IOException e) {
throw new SearchException(shardTarget, "failed to create SearchContextHighlighter", e);
}
}
if (source.scriptFields() != null && source.size() != 0) {
int maxAllowedScriptFields = context.mapperService().getIndexSettings().getMaxScriptFields();
if (source.scriptFields().size() > maxAllowedScriptFields) {
throw new IllegalArgumentException("Trying to retrieve too many script_fields. Must be less than or equal to: [" + maxAllowedScriptFields + "] but was [" + source.scriptFields().size() + "]. This limit can be set by changing the [" + IndexSettings.MAX_SCRIPT_FIELDS_SETTING.getKey() + "] index level setting.");
}
for (org.opensearch.search.builder.SearchSourceBuilder.ScriptField field : source.scriptFields()) {
FieldScript.Factory factory = scriptService.compile(field.script(), FieldScript.CONTEXT);
SearchLookup lookup = context.getQueryShardContext().lookup();
FieldScript.LeafFactory searchScript = factory.newFactory(field.script().getParams(), lookup);
context.scriptFields().add(new ScriptField(field.fieldName(), searchScript, field.ignoreFailure()));
}
}
if (source.ext() != null) {
for (SearchExtBuilder searchExtBuilder : source.ext()) {
context.addSearchExt(searchExtBuilder);
}
}
if (source.version() != null) {
context.version(source.version());
}
if (source.seqNoAndPrimaryTerm() != null) {
context.seqNoAndPrimaryTerm(source.seqNoAndPrimaryTerm());
}
if (source.stats() != null) {
context.groupStats(source.stats());
}
if (CollectionUtils.isEmpty(source.searchAfter()) == false) {
if (context.scrollContext() != null) {
throw new SearchException(shardTarget, "`search_after` cannot be used in a scroll context.");
}
if (context.from() > 0) {
throw new SearchException(shardTarget, "`from` parameter must be set to 0 when `search_after` is used.");
}
FieldDoc fieldDoc = SearchAfterBuilder.buildFieldDoc(context.sort(), source.searchAfter());
context.searchAfter(fieldDoc);
}
if (source.slice() != null) {
if (context.scrollContext() == null) {
throw new SearchException(shardTarget, "`slice` cannot be used outside of a scroll context");
}
context.sliceBuilder(source.slice());
}
if (source.storedFields() != null) {
if (source.storedFields().fetchFields() == false) {
if (context.sourceRequested()) {
throw new SearchException(shardTarget, "[stored_fields] cannot be disabled if [_source] is requested");
}
if (context.fetchFieldsContext() != null) {
throw new SearchException(shardTarget, "[stored_fields] cannot be disabled when using the [fields] option");
}
}
context.storedFieldsContext(source.storedFields());
}
if (source.collapse() != null) {
if (context.scrollContext() != null) {
throw new SearchException(shardTarget, "cannot use `collapse` in a scroll context");
}
if (context.searchAfter() != null) {
throw new SearchException(shardTarget, "cannot use `collapse` in conjunction with `search_after`");
}
if (context.rescore() != null && context.rescore().isEmpty() == false) {
throw new SearchException(shardTarget, "cannot use `collapse` in conjunction with `rescore`");
}
final CollapseContext collapseContext = source.collapse().build(queryShardContext);
context.collapse(collapseContext);
}
}
use of org.opensearch.search.fetch.subphase.highlight.HighlightBuilder in project kestra by kestra-io.
the class ElasticSearchFlowRepository method findSourceCode.
@Override
public ArrayListTotal<SearchResult<Flow>> findSourceCode(String query, Pageable pageable) {
BoolQueryBuilder bool = this.defaultFilter().must(QueryBuilders.queryStringQuery(query).field("sourceCode"));
SearchSourceBuilder sourceBuilder = this.searchSource(bool, Optional.empty(), pageable);
sourceBuilder.fetchSource("*", "sourceCode");
sourceBuilder.highlighter(new HighlightBuilder().preTags("[mark]").postTags("[/mark]").field("sourceCode"));
SearchRequest searchRequest = searchRequest(INDEX_NAME, sourceBuilder, false);
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return new ArrayListTotal<>(Arrays.stream(searchResponse.getHits().getHits()).map(documentFields -> {
try {
return new SearchResult<>(MAPPER.readValue(documentFields.getSourceAsString(), this.cls), documentFields.getHighlightFields().get("sourceCode") != null ? Arrays.stream(documentFields.getHighlightFields().get("sourceCode").getFragments()).map(Text::string).collect(Collectors.toList()) : Collections.emptyList());
} catch (IOException e) {
throw new RuntimeException(e);
}
}).collect(Collectors.toList()), searchResponse.getHits().getTotalHits().value);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
Aggregations