Examples with StupidBackoff org.opensearch.search.suggest.phrase.StupidBackoff used on opensource projects

Example 1 with StupidBackoff

use of org.opensearch.search.suggest.phrase.StupidBackoff in project OpenSearch by opensearch-project.

the class SuggestSearchIT method testBasicPhraseSuggest.

public void testBasicPhraseSuggest() throws IOException, URISyntaxException {
    CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put("index.analysis.analyzer.body.tokenizer", "standard").putList("index.analysis.analyzer.body.filter", "lowercase").put("index.analysis.analyzer.bigram.tokenizer", "standard").putList("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase").put("index.analysis.filter.my_shingle.type", "shingle").put("index.analysis.filter.my_shingle.output_unigrams", false).put("index.analysis.filter.my_shingle.min_shingle_size", 2).put("index.analysis.filter.my_shingle.max_shingle_size", 2).put("index.number_of_shards", 1));
    XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("body").field("type", "text").field("analyzer", "body").endObject().startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject().endObject().endObject().endObject();
    assertAcked(builder.addMapping("type1", mapping));
    ensureGreen();
    String[] strings = new String[] { "Arthur, King of the Britons", "Sir Lancelot the Brave", "Patsy, Arthur's Servant", "Sir Robin the Not-Quite-So-Brave-as-Sir-Lancelot", "Sir Bedevere the Wise", "Sir Galahad the Pure", "Miss Islington, the Witch", "Zoot", "Leader of Robin's Minstrels", "Old Crone", "Frank, the Historian", "Frank's Wife", "Dr. Piglet", "Dr. Winston", "Sir Robin (Stand-in)", "Knight Who Says Ni", "Police sergeant who stops the film" };
    for (String line : strings) {
        index("test", "type1", line, "body", line, "bigram", line);
    }
    refresh();
    PhraseSuggestionBuilder phraseSuggest = phraseSuggestion("bigram").gramSize(2).analyzer("body").addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always")).size(1);
    Suggest searchSuggest = searchSuggest("Frank's Wise", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "frank's wife");
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Frank's Wise"));
    phraseSuggest.realWordErrorLikelihood(0.95f);
    searchSuggest = searchSuggest("Artur, Kinh of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // Check the "text" field this one time.
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Artur, Kinh of the Britons"));
    // Ask for highlighting
    phraseSuggest.highlight("<em>", "</em>");
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getOptions().get(0).getHighlighted().string(), equalTo("<em>arthur</em> king of the <em>britons</em>"));
    // pass in a correct phrase
    phraseSuggest.highlight(null, null).confidence(0f).size(1).maxErrors(0.5f);
    searchSuggest = searchSuggest("Arthur, King of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // pass in a correct phrase - set confidence to 2
    phraseSuggest.confidence(2f);
    searchSuggest = searchSuggest("Arthur, King of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    // pass in a correct phrase - set confidence to 0.99
    phraseSuggest.confidence(0.99f);
    searchSuggest = searchSuggest("Arthur, King of the Britons", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // set all mass to trigrams (not indexed)
    phraseSuggest.clearCandidateGenerators().addCandidateGenerator(candidateGenerator("body").minWordLength(1).suggestMode("always")).smoothingModel(new LinearInterpolation(1, 0, 0));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    // set all mass to bigrams
    phraseSuggest.smoothingModel(new LinearInterpolation(0, 1, 0));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // distribute mass
    phraseSuggest.smoothingModel(new LinearInterpolation(0.4, 0.4, 0.2));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    searchSuggest = searchSuggest("Frank's Wise", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "frank's wife");
    // try all smoothing methods
    phraseSuggest.smoothingModel(new LinearInterpolation(0.4, 0.4, 0.2));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    phraseSuggest.smoothingModel(new Laplace(0.2));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    phraseSuggest.smoothingModel(new StupidBackoff(0.1));
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "arthur king of the britons");
    // check tokenLimit
    phraseSuggest.smoothingModel(null).tokenLimit(4);
    searchSuggest = searchSuggest("Artur, King of the Britns", "simple_phrase", phraseSuggest);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    phraseSuggest.tokenLimit(15).smoothingModel(new StupidBackoff(0.1));
    searchSuggest = searchSuggest("Sir Bedever the Wife Sir Bedever the Wife Sir Bedever the Wife", "simple_phrase", phraseSuggest);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "sir bedevere the wise sir bedevere the wise sir bedevere the wise");
    // Check the name this time because we're repeating it which is funky
    assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getText().string(), equalTo("Sir Bedever the Wife Sir Bedever the Wife Sir Bedever the Wife"));
}

Example 2 with StupidBackoff

use of org.opensearch.search.suggest.phrase.StupidBackoff in project OpenSearch by opensearch-project.

the class SuggestSearchIT method testSizeParam.

public void testSizeParam() throws IOException {
    CreateIndexRequestBuilder builder = prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put("index.analysis.analyzer.body.tokenizer", "standard").putList("index.analysis.analyzer.body.filter", "lowercase").put("index.analysis.analyzer.bigram.tokenizer", "standard").putList("index.analysis.analyzer.bigram.filter", "my_shingle", "lowercase").put("index.analysis.filter.my_shingle.type", "shingle").put("index.analysis.filter.my_shingle.output_unigrams", false).put("index.analysis.filter.my_shingle.min_shingle_size", 2).put("index.analysis.filter.my_shingle.max_shingle_size", 2));
    XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("body").field("type", "text").field("analyzer", "body").endObject().startObject("bigram").field("type", "text").field("analyzer", "bigram").endObject().endObject().endObject().endObject();
    assertAcked(builder.addMapping("type1", mapping));
    ensureGreen();
    String line = "xorr the god jewel";
    index("test", "type1", "1", "body", line, "bigram", line);
    line = "I got it this time";
    index("test", "type1", "2", "body", line, "bigram", line);
    refresh();
    PhraseSuggestionBuilder phraseSuggestion = phraseSuggestion("bigram").realWordErrorLikelihood(0.95f).gramSize(2).analyzer("body").addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(1).accuracy(0.1f)).smoothingModel(new StupidBackoff(0.1)).maxErrors(1.0f).size(5);
    Suggest searchSuggest = searchSuggest("Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
    assertSuggestionSize(searchSuggest, 0, 0, "simple_phrase");
    // we allow a size of 2 now on the shard generator level so "god" will be found since it's LD2
    phraseSuggestion.clearCandidateGenerators().addCandidateGenerator(candidateGenerator("body").minWordLength(1).prefixLength(1).suggestMode("always").size(2).accuracy(0.1f));
    searchSuggest = searchSuggest("Xorr the Gut-Jewel", "simple_phrase", phraseSuggestion);
    assertSuggestion(searchSuggest, 0, "simple_phrase", "xorr the god jewel");
}