Examples with CreateIndexRequestBuilder org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder used on opensource projects

Example 1 with CreateIndexRequestBuilder

use of org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder in project OpenSearch by opensearch-project.

the class MatchPhraseQueryIT method setUp.

@Before
public void setUp() throws Exception {
    super.setUp();
    CreateIndexRequestBuilder createIndexRequest = prepareCreate(INDEX).setSettings(Settings.builder().put(indexSettings()).put("index.analysis.analyzer.standard_stopwords.type", "standard").putList("index.analysis.analyzer.standard_stopwords.stopwords", "of", "the", "who"));
    assertAcked(createIndexRequest);
    ensureGreen();
}

Example 2 with CreateIndexRequestBuilder

use of org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder in project OpenSearch by opensearch-project.

the class SuggestSearchIT method testPrefixLength.

public void testPrefixLength() throws IOException {
    CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put("index.analysis.analyzer.body.tokenizer", "standard").putList("index.analysis.analyzer.body.filter", "lowercase").put("index.analysis.analyzer.bigram.tokenizer", "standard").putList("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase").put("index.analysis.filter.my_shingle.type", "shingle").put("index.analysis.filter.my_shingle.output_unigrams", false).put("index.analysis.filter.my_shingle.min_shingle_size", 2).put("index.analysis.filter.my_shingle.max_shingle_size", 2));
    XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("body").field("type", "text").field("analyzer", "body").endObject().startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject().endObject().endObject().endObject();
    assertAcked(builder.addMapping("type1", mapping));
    ensureGreen();
    index("test", "type1", "1", "body", "hello world");
    index("test", "type1", "2", "body", "hello world");
    index("test", "type1", "3", "body", "hello words");
    refresh();
    Suggest searchSuggest = searchSuggest("hello word", "simple_phrase", phraseSuggestion("body").addCandidateGenerator(candidateGenerator("body").prefixLength(4).minWordLength(1).suggestMode("always")).size(1).confidence(1.0f));
    assertSuggestion(searchSuggest, 0, "simple_phrase", "hello words");
    searchSuggest = searchSuggest("hello word", "simple_phrase", phraseSuggestion("body").addCandidateGenerator(candidateGenerator("body").prefixLength(2).minWordLength(1).suggestMode("always")).size(1).confidence(1.0f));
    assertSuggestion(searchSuggest, 0, "simple_phrase", "hello world");
}

Example 3 with CreateIndexRequestBuilder

use of org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder in project OpenSearch by opensearch-project.

the class SuggestSearchIT method testSearchForRarePhrase.

/**
 * Searching for a rare phrase shouldn't provide any suggestions if confidence > 1.  This was possible before we rechecked the cutoff
 * score during the reduce phase.  Failures don't occur every time - maybe two out of five tries but we don't repeat it to save time.
 */
public void testSearchForRarePhrase() throws IOException {
    // If there isn't enough chaf per shard then shards can become unbalanced, making the cutoff recheck this is testing do more harm
    // then good.
    int chafPerShard = 100;
    CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put("index.analysis.analyzer.body.tokenizer", "standard").putList("index.analysis.analyzer.body.filter", "lowercase", "my_shingle").put("index.analysis.filter.my_shingle.type", "shingle").put("index.analysis.filter.my_shingle.output_unigrams", true).put("index.analysis.filter.my_shingle.min_shingle_size", 2).put("index.analysis.filter.my_shingle.max_shingle_size", 2));
    XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("body").field("type", "text").field("analyzer", "body").endObject().endObject().endObject().endObject();
    assertAcked(builder.addMapping("type1", mapping));
    ensureGreen();
    NumShards test = getNumShards("test");
    List<String> phrases = new ArrayList<>();
    Collections.addAll(phrases, "nobel prize", "noble gases", "somethingelse prize", "pride and joy", "notes are fun");
    for (int i = 0; i < 8; i++) {
        phrases.add("noble somethingelse" + i);
    }
    for (int i = 0; i < test.numPrimaries * chafPerShard; i++) {
        phrases.add("chaff" + i);
    }
    for (String phrase : phrases) {
        index("test", "type1", phrase, "body", phrase);
    }
    refresh();
    Suggest searchSuggest = searchSuggest("nobel prize", "simple_phrase", phraseSuggestion("body").addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f)).confidence(2f).maxErrors(5f).size(1));
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    searchSuggest = searchSuggest("noble prize", "simple_phrase", phraseSuggestion("body").addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always").maxTermFreq(.99f)).confidence(2f).maxErrors(5f).size(1));
    assertSuggestion(searchSuggest, 0, 0, "simple_phrase", "nobel prize");
}

Example 4 with CreateIndexRequestBuilder

use of org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder in project OpenSearch by opensearch-project.

the class SuggestSearchIT method testBasicPhraseSuggest.

public void testBasicPhraseSuggest() throws IOException, URISyntaxException {
    CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put("index.analysis.analyzer.body.tokenizer", "standard").putList("index.analysis.analyzer.body.filter", "lowercase").put("index.analysis.analyzer.bigram.tokenizer", "standard").putList("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase").put("index.analysis.filter.my_shingle.type", "shingle").put("index.analysis.filter.my_shingle.output_unigrams", false).put("index.analysis.filter.my_shingle.min_shingle_size", 2).put("index.analysis.filter.my_shingle.max_shingle_size", 2).put("index.number_of_shards", 1));
    XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("body").field("type", "text").field("analyzer", "body").endObject().startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject().endObject().endObject().endObject();
    assertAcked(builder.addMapping("type1", mapping));
    ensureGreen();
    String[] strings = new String[] { "Arthur, King of the Britons", "Sir Lancelot the Brave", "Patsy, Arthur's Servant", "Sir Robin the Not-Quite-So-Brave-as-Sir-Lancelot", "Sir Bedevere the Wise", "Sir Galahad the Pure", "Miss Islington, the Witch", "Zoot", "Leader of Robin's Minstrels", "Old Crone", "Frank, the Historian", "Frank's Wife", "Dr. Piglet", "Dr. Winston", "Sir Robin (Stand-in)", "Knight Who Says Ni", "Police sergeant who stops the film" };
    for (String line : strings) {
        index("test", "type1", line, "body", line, "bigram", line);
    }
    refresh();
    PhraseSuggestionBuilder phraseSuggest = phraseSuggestion("bigram").gramSize(2).analyzer("body").addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always")).size(1);
    Suggest searchSuggest = searchSuggest("Frank's Wise", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "frank's wife");
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Frank's Wise"));
    phraseSuggest.realWordErrorLikelihood(0.95f);
    searchSuggest = searchSuggest("Artur, Kinh of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // Check the "text" field this one time.
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Artur, Kinh of the Britons"));
    // Ask for highlighting
    phraseSuggest.highlight("<em>", "</em>");
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getOptions().get(0).getHighlighted().string(), equalTo("<em>arthur</em> king of the <em>britons</em>"));
    // pass in a correct phrase
    phraseSuggest.highlight(null, null).confidence(0f).size(1).maxErrors(0.5f);
    searchSuggest = searchSuggest("Arthur, King of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // pass in a correct phrase - set confidence to 2
    phraseSuggest.confidence(2f);
    searchSuggest = searchSuggest("Arthur, King of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    // pass in a correct phrase - set confidence to 0.99
    phraseSuggest.confidence(0.99f);
    searchSuggest = searchSuggest("Arthur, King of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // set all mass to trigrams (not indexed)
    phraseSuggest.clearCandidateGenerators().addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always")).smoothingModel(new LinearInterpolation(1, 0, 0));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    // set all mass to bigrams
    phraseSuggest.smoothingModel(new LinearInterpolation(0, 1, 0));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // distribute mass
    phraseSuggest.smoothingModel(new LinearInterpolation(0.4, 0.4, 0.2));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    searchSuggest = searchSuggest("Frank's Wise", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "frank's wife");
    // try all smoothing methods
    phraseSuggest.smoothingModel(new LinearInterpolation(0.4, 0.4, 0.2));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    phraseSuggest.smoothingModel(new Laplace(0.2));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    phraseSuggest.smoothingModel(new StupidBackoff(0.1));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // check tokenLimit
    phraseSuggest.smoothingModel(null).tokenLimit(4);
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
    searchSuggest = searchSuggest("Sir Bedever the Wife Sir Bedever the Wife Sir Bedever the Wife", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "sir bedevere the wise sir bedevere the wise sir bedevere the wise");
    // Check the name this time because we're repeating it which is funky
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Sir Bedever the Wife Sir Bedever the Wife Sir Bedever the Wife"));
}

Example 5 with CreateIndexRequestBuilder

use of org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder in project OpenSearch by opensearch-project.

the class SuggestSearchIT method testSizeParam.

public void testSizeParam() throws IOException {
    CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put("index.analysis.analyzer.body.tokenizer", "standard").putList("index.analysis.analyzer.body.filter", "lowercase").put("index.analysis.analyzer.bigram.tokenizer", "standard").putList("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase").put("index.analysis.filter.my_shingle.type", "shingle").put("index.analysis.filter.my_shingle.output_unigrams", false).put("index.analysis.filter.my_shingle.min_shingle_size", 2).put("index.analysis.filter.my_shingle.max_shingle_size", 2));
    XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("body").field("type", "text").field("analyzer", "body").endObject().startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject().endObject().endObject().endObject();
    assertAcked(builder.addMapping("type1", mapping));
    ensureGreen();
    String line = "xorr the god jewel";
    index("test", "type1", "1", "body", line, "bigram", line);
    line = "I got it this time";
    index("test", "type1", "2", "body", line, "bigram", line);
    refresh();
    PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram").realWordErrorLikelihood(0.95f).gramSize(2).analyzer("body").addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f)).smoothingModel(new StupidBackoff(0.1)).maxErrors(1.0f).size(5);
    Suggest searchSuggest = searchSuggest("Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    // we allow a size of 2 now on the shard generator level so "god" will be found since it's LD2
    phraseSuggestion.clearCandidateGenerators().addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(2).accuracy(0.1f));
    searchSuggest = searchSuggest("Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
}