use of org.opensearch.common.settings.Settings.Builder in project OpenSearch by opensearch-project.
the class HighlighterSearchIT method checkMatchedFieldsCase.
private void checkMatchedFieldsCase(boolean requireFieldMatch) throws Exception {
Settings.Builder settings = Settings.builder();
settings.put(indexSettings());
settings.put("index.analysis.analyzer.mock_english.tokenizer", "standard");
settings.put("index.analysis.analyzer.mock_english.filter", "mock_snowball");
assertAcked(prepareCreate("test").setSettings(settings).addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("foo").field("type", "text").field("term_vector", "with_positions_offsets").field("store", true).field("analyzer", "mock_english").startObject("fields").startObject("plain").field("type", "text").field("term_vector", "with_positions_offsets").field("analyzer", "standard").endObject().endObject().endObject().startObject("bar").field("type", "text").field("term_vector", "with_positions_offsets").field("store", true).field("analyzer", "mock_english").startObject("fields").startObject("plain").field("type", "text").field("term_vector", "with_positions_offsets").field("analyzer", "standard").endObject().endObject().endObject().endObject().endObject().endObject()));
ensureGreen();
index("test", "type1", "1", "foo", "running with scissors");
index("test", "type1", "2", "foo", "cat cat junk junk junk junk junk junk junk cats junk junk", "bar", "cat cat junk junk junk junk junk junk junk cats junk junk");
index("test", "type1", "3", "foo", "weird", "bar", "result");
refresh();
Field fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh").requireFieldMatch(requireFieldMatch);
SearchRequestBuilder req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField));
// First check highlighting without any matched fields set
SearchResponse resp = req.setQuery(queryStringQuery("running scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
// And that matching a subfield doesn't automatically highlight it
resp = req.setQuery(queryStringQuery("foo.plain:running scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("running with <em>scissors</em>"));
// Add the subfield to the list of matched fields but don't match it. Everything should still work
// like before we added it.
fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh").requireFieldMatch(requireFieldMatch);
fooField.matchedFields("foo", "foo.plain");
req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField));
resp = req.setQuery(queryStringQuery("running scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
// Now make half the matches come from the stored field and half from just a matched field.
resp = req.setQuery(queryStringQuery("foo.plain:running scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
// Now remove the stored field from the matched field list. That should work too.
fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh").requireFieldMatch(requireFieldMatch);
fooField.matchedFields("foo.plain");
req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField));
resp = req.setQuery(queryStringQuery("foo.plain:running scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with scissors"));
// Now make sure boosted fields don't blow up when matched fields is both the subfield and stored field.
fooField = new Field("foo").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh").requireFieldMatch(requireFieldMatch);
fooField.matchedFields("foo", "foo.plain");
req = client().prepareSearch("test").highlighter(new HighlightBuilder().field(fooField));
resp = req.setQuery(queryStringQuery("foo.plain:running^5 scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
// Now just all matches are against the matched field. This still returns highlighting.
resp = req.setQuery(queryStringQuery("foo.plain:running foo.plain:scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
// And all matched field via the queryString's field parameter, just in case
resp = req.setQuery(queryStringQuery("running scissors").field("foo.plain")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
// Finding the same string two ways is ok too
resp = req.setQuery(queryStringQuery("run foo.plain:running^5 scissors").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
// But we use the best found score when sorting fragments
resp = req.setQuery(queryStringQuery("cats foo.plain:cats^5").field("foo")).get();
assertHighlight(resp, 0, "foo", 0, equalTo("junk junk <em>cats</em> junk junk"));
// which can also be written by searching on the subfield
resp = req.setQuery(queryStringQuery("cats").field("foo").field("foo.plain", 5)).get();
assertHighlight(resp, 0, "foo", 0, equalTo("junk junk <em>cats</em> junk junk"));
// Speaking of two fields, you can have two fields, only one of which has matchedFields enabled
QueryBuilder twoFieldsQuery = queryStringQuery("cats").field("foo").field("foo.plain", 5).field("bar").field("bar.plain", 5);
Field barField = new Field("bar").numOfFragments(1).order("score").fragmentSize(25).highlighterType("fvh").requireFieldMatch(requireFieldMatch);
resp = req.setQuery(twoFieldsQuery).highlighter(new HighlightBuilder().field(fooField).field(barField)).get();
assertHighlight(resp, 0, "foo", 0, equalTo("junk junk <em>cats</em> junk junk"));
assertHighlight(resp, 0, "bar", 0, equalTo("<em>cat</em> <em>cat</em> junk junk junk junk"));
// And you can enable matchedField highlighting on both
barField.matchedFields("bar", "bar.plain");
resp = req.setQuery(twoFieldsQuery).highlighter(new HighlightBuilder().field(fooField).field(barField)).get();
assertHighlight(resp, 0, "foo", 0, equalTo("junk junk <em>cats</em> junk junk"));
assertHighlight(resp, 0, "bar", 0, equalTo("junk junk <em>cats</em> junk junk"));
// Setting a matchedField that isn't searched/doesn't exist is simply ignored.
barField.matchedFields("bar", "candy");
resp = req.setQuery(twoFieldsQuery).highlighter(new HighlightBuilder().field(fooField).field(barField)).get();
assertHighlight(resp, 0, "foo", 0, equalTo("junk junk <em>cats</em> junk junk"));
assertHighlight(resp, 0, "bar", 0, equalTo("<em>cat</em> <em>cat</em> junk junk junk junk"));
// If the stored field doesn't have a value it doesn't matter what you match, you get nothing.
barField.matchedFields("bar", "foo.plain");
resp = req.setQuery(queryStringQuery("running scissors").field("foo.plain").field("bar")).highlighter(new HighlightBuilder().field(fooField).field(barField)).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
assertThat(resp.getHits().getAt(0).getHighlightFields(), not(hasKey("bar")));
// If the stored field is found but the matched field isn't then you don't get a result either.
fooField.matchedFields("bar.plain");
resp = req.setQuery(queryStringQuery("running scissors").field("foo").field("foo.plain").field("bar").field("bar.plain")).highlighter(new HighlightBuilder().field(fooField).field(barField)).get();
assertThat(resp.getHits().getAt(0).getHighlightFields(), not(hasKey("foo")));
// But if you add the stored field to the list of matched fields then you'll get a result again
fooField.matchedFields("foo", "bar.plain");
resp = req.setQuery(queryStringQuery("running scissors").field("foo").field("foo.plain").field("bar").field("bar.plain")).highlighter(new HighlightBuilder().field(fooField).field(barField)).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>running</em> with <em>scissors</em>"));
assertThat(resp.getHits().getAt(0).getHighlightFields(), not(hasKey("bar")));
// You _can_ highlight fields that aren't subfields of one another.
resp = req.setQuery(queryStringQuery("weird").field("foo").field("foo.plain").field("bar").field("bar.plain")).highlighter(new HighlightBuilder().field(fooField).field(barField)).get();
assertHighlight(resp, 0, "foo", 0, equalTo("<em>weird</em>"));
assertHighlight(resp, 0, "bar", 0, equalTo("<em>resul</em>t"));
assertFailures(req.setQuery(queryStringQuery("result").field("foo").field("foo.plain").field("bar").field("bar.plain")), RestStatus.INTERNAL_SERVER_ERROR, containsString("IndexOutOfBoundsException"));
}
use of org.opensearch.common.settings.Settings.Builder in project OpenSearch by opensearch-project.
the class SearchWithRandomExceptionsIT method testRandomExceptions.
public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("test").field("type", "keyword").endObject().endObject().endObject().endObject());
final double lowLevelRate;
final double topLevelRate;
if (frequently()) {
if (randomBoolean()) {
if (randomBoolean()) {
lowLevelRate = 1.0 / between(2, 10);
topLevelRate = 0.0d;
} else {
topLevelRate = 1.0 / between(2, 10);
lowLevelRate = 0.0d;
}
} else {
lowLevelRate = 1.0 / between(2, 10);
topLevelRate = 1.0 / between(2, 10);
}
} else {
// rarely no exception
topLevelRate = 0d;
lowLevelRate = 0d;
}
Builder settings = Settings.builder().put(indexSettings()).put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate).put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate).put(MockEngineSupport.WRAP_READER_RATIO.getKey(), 1.0d);
logger.info("creating index: [test] using settings: [{}]", settings.build());
assertAcked(prepareCreate("test").setSettings(settings).addMapping("type", mapping, XContentType.JSON));
ensureSearchable();
final int numDocs = between(10, 100);
int numCreated = 0;
boolean[] added = new boolean[numDocs];
for (int i = 0; i < numDocs; i++) {
try {
IndexResponse indexResponse = client().prepareIndex("test").setId("" + i).setTimeout(TimeValue.timeValueSeconds(1)).setSource("test", English.intToEnglish(i)).get();
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) {
numCreated++;
added[i] = true;
}
} catch (OpenSearchException ex) {
}
}
logger.info("Start Refresh");
// don't assert on failures here
RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().get();
final boolean refreshFailed = refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
logger.info("Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ", refreshFailed, refreshResponse.getFailedShards(), refreshResponse.getShardFailures().length, refreshResponse.getSuccessfulShards(), refreshResponse.getTotalShards());
NumShards test = getNumShards("test");
final int numSearches = scaledRandomIntBetween(100, 200);
// we don't check anything here really just making sure we don't leave any open files or a broken index behind.
for (int i = 0; i < numSearches; i++) {
try {
int docToQuery = between(0, numDocs - 1);
int expectedResults = added[docToQuery] ? 1 : 0;
logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery))).setSize(expectedResults).get();
logger.info("Successful shards: [{}] numShards: [{}]", searchResponse.getSuccessfulShards(), test.numPrimaries);
if (searchResponse.getSuccessfulShards() == test.numPrimaries && !refreshFailed) {
assertResultsAndLogOnFailure(expectedResults, searchResponse);
}
// check match all
searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).setSize(numCreated).addSort("_id", SortOrder.ASC).get();
logger.info("Match all Successful shards: [{}] numShards: [{}]", searchResponse.getSuccessfulShards(), test.numPrimaries);
if (searchResponse.getSuccessfulShards() == test.numPrimaries && !refreshFailed) {
assertResultsAndLogOnFailure(numCreated, searchResponse);
}
} catch (SearchPhaseExecutionException ex) {
logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
}
}
}
use of org.opensearch.common.settings.Settings.Builder in project OpenSearch by opensearch-project.
the class QueryRescorerIT method testMoreDocs.
public void testMoreDocs() throws Exception {
Builder builder = Settings.builder();
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("field1").field("type", "text").field("analyzer", "whitespace").endObject().endObject().endObject().endObject();
assertAcked(client().admin().indices().prepareCreate("test").addMapping("type1", mapping).setSettings(builder.put("index.number_of_shards", 1)));
client().prepareIndex("test").setId("1").setSource("field1", "massachusetts avenue boston massachusetts").get();
client().prepareIndex("test").setId("2").setSource("field1", "lexington avenue boston massachusetts").get();
client().prepareIndex("test").setId("3").setSource("field1", "boston avenue lexington massachusetts").get();
client().admin().indices().prepareRefresh("test").get();
client().prepareIndex("test").setId("4").setSource("field1", "boston road lexington massachusetts").get();
client().prepareIndex("test").setId("5").setSource("field1", "lexington street lexington massachusetts").get();
client().prepareIndex("test").setId("6").setSource("field1", "massachusetts avenue lexington massachusetts").get();
client().prepareIndex("test").setId("7").setSource("field1", "bosten street san franciso california").get();
client().admin().indices().prepareRefresh("test").get();
client().prepareIndex("test").setId("8").setSource("field1", "hollywood boulevard los angeles california").get();
client().prepareIndex("test").setId("9").setSource("field1", "1st street boston massachussetts").get();
client().prepareIndex("test").setId("10").setSource("field1", "1st street boston massachusetts").get();
client().admin().indices().prepareRefresh("test").get();
client().prepareIndex("test").setId("11").setSource("field1", "2st street boston massachusetts").get();
client().prepareIndex("test").setId("12").setSource("field1", "3st street boston massachusetts").get();
client().admin().indices().prepareRefresh("test").get();
SearchResponse searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "lexington avenue massachusetts").operator(Operator.OR)).setFrom(0).setSize(5).setRescorer(new QueryRescorerBuilder(matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)).setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 20).get();
assertThat(searchResponse.getHits().getHits().length, equalTo(5));
assertHitCount(searchResponse, 9);
assertFirstHit(searchResponse, hasId("2"));
assertSecondHit(searchResponse, hasId("6"));
assertThirdHit(searchResponse, hasId("3"));
searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "lexington avenue massachusetts").operator(Operator.OR)).setFrom(0).setSize(5).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setRescorer(new QueryRescorerBuilder(matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)).setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 20).get();
assertThat(searchResponse.getHits().getHits().length, equalTo(5));
assertHitCount(searchResponse, 9);
assertThat(searchResponse.getHits().getMaxScore(), equalTo(searchResponse.getHits().getHits()[0].getScore()));
assertFirstHit(searchResponse, hasId("2"));
assertSecondHit(searchResponse, hasId("6"));
assertThirdHit(searchResponse, hasId("3"));
// Make sure non-zero from works:
searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchQuery("field1", "lexington avenue massachusetts").operator(Operator.OR)).setFrom(2).setSize(5).setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setRescorer(new QueryRescorerBuilder(matchPhraseQuery("field1", "lexington avenue massachusetts").slop(3)).setQueryWeight(0.6f).setRescoreQueryWeight(2.0f), 20).get();
assertThat(searchResponse.getHits().getHits().length, equalTo(5));
assertHitCount(searchResponse, 9);
assertThat(searchResponse.getHits().getMaxScore(), greaterThan(searchResponse.getHits().getHits()[0].getScore()));
assertFirstHit(searchResponse, hasId("3"));
}
use of org.opensearch.common.settings.Settings.Builder in project OpenSearch by opensearch-project.
the class QueryRescorerIT method testFromSize.
// #11277
public void testFromSize() throws Exception {
Builder settings = Settings.builder();
settings.put(SETTING_NUMBER_OF_SHARDS, 1);
settings.put(SETTING_NUMBER_OF_REPLICAS, 0);
assertAcked(prepareCreate("test").setSettings(settings));
for (int i = 0; i < 5; i++) {
client().prepareIndex("test").setId("" + i).setSource("text", "hello world").get();
}
refresh();
SearchRequestBuilder request = client().prepareSearch();
request.setQuery(QueryBuilders.termQuery("text", "hello"));
request.setFrom(1);
request.setSize(4);
request.addRescorer(new QueryRescorerBuilder(matchAllQuery()), 50);
assertEquals(4, request.get().getHits().getHits().length);
}
use of org.opensearch.common.settings.Settings.Builder in project OpenSearch by opensearch-project.
the class StopTokenFilterTests method testCorrectPositionIncrementSetting.
public void testCorrectPositionIncrementSetting() throws IOException {
Builder builder = Settings.builder().put("index.analysis.filter.my_stop.type", "stop");
if (random().nextBoolean()) {
builder.put("index.analysis.filter.my_stop.version", Version.LATEST);
} else {
// don't specify
}
builder.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString());
OpenSearchTestCase.TestAnalysis analysis = AnalysisTestsHelper.createTestAnalysisFromSettings(builder.build());
TokenFilterFactory tokenFilter = analysis.tokenFilter.get("my_stop");
assertThat(tokenFilter, instanceOf(StopTokenFilterFactory.class));
Tokenizer tokenizer = new WhitespaceTokenizer();
tokenizer.setReader(new StringReader("foo bar"));
TokenStream create = tokenFilter.create(tokenizer);
assertThat(create, instanceOf(StopFilter.class));
}
Aggregations