Search in sources :

Example 6 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class NGramTestCase method requireThatAccessorsWork.

@Test
public void requireThatAccessorsWork() {
    Linguistics linguistics = new SimpleLinguistics();
    NGramExpression exp = new NGramExpression(linguistics, 69);
    assertSame(linguistics, exp.getLinguistics());
    assertEquals(69, exp.getGramSize());
}
Also used : SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) Linguistics(com.yahoo.language.Linguistics) SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) Test(org.junit.Test)

Example 7 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class TokenizerTestCase method testOutsideBMPCodepoints.

@Test
public void testOutsideBMPCodepoints() {
    Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
    List<?> tokens = tokenizer.tokenize("\ud841\udd47");
    assertEquals(new Token(WORD, "\ud841\udd47"), tokens.get(0));
}
Also used : SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) Token(com.yahoo.prelude.query.parser.Token) Tokenizer(com.yahoo.prelude.query.parser.Tokenizer) Test(org.junit.Test)

Example 8 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class TokenizerTestCase method testOneSpecialToken.

@Test
public void testOneSpecialToken() {
    Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
    tokenizer.setSpecialTokens(createSpecialTokens());
    List<?> tokens = tokenizer.tokenize("c++ lovers, please apply");
    assertEquals(new Token(WORD, "c++"), tokens.get(0));
}
Also used : SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) Token(com.yahoo.prelude.query.parser.Token) Tokenizer(com.yahoo.prelude.query.parser.Tokenizer) Test(org.junit.Test)

Example 9 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class TokenizerTestCase method testSingleQuoteAsWordCharacter.

@Test
public void testSingleQuoteAsWordCharacter() {
    Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
    tokenizer.setSpecialTokens(createSpecialTokens());
    List<?> tokens = tokenizer.tokenize("drive (to hwy88, 88) +or language:en nalle:a'a ugcapi_1 'a' 'a a'");
    assertEquals(new Token(WORD, "drive"), tokens.get(0));
    assertEquals(new Token(SPACE, " "), tokens.get(1));
    assertEquals(new Token(LBRACE, "("), tokens.get(2));
    assertEquals(new Token(WORD, "to"), tokens.get(3));
    assertEquals(new Token(SPACE, " "), tokens.get(4));
    assertEquals(new Token(WORD, "hwy88"), tokens.get(5));
    assertEquals(new Token(COMMA, ","), tokens.get(6));
    assertEquals(new Token(SPACE, " "), tokens.get(7));
    assertEquals(new Token(NUMBER, "88"), tokens.get(8));
    assertEquals(new Token(RBRACE, ")"), tokens.get(9));
    assertEquals(new Token(SPACE, " "), tokens.get(10));
    assertEquals(new Token(PLUS, "+"), tokens.get(11));
    assertEquals(new Token(WORD, "or"), tokens.get(12));
    assertEquals(new Token(SPACE, " "), tokens.get(13));
    assertEquals(new Token(WORD, "language"), tokens.get(14));
    assertEquals(new Token(COLON, ":"), tokens.get(15));
    assertEquals(new Token(WORD, "en"), tokens.get(16));
    assertEquals(new Token(SPACE, " "), tokens.get(17));
    assertEquals(new Token(WORD, "nalle"), tokens.get(18));
    assertEquals(new Token(COLON, ":"), tokens.get(19));
    assertEquals(new Token(WORD, "a'a"), tokens.get(20));
    assertEquals(new Token(SPACE, " "), tokens.get(21));
    assertEquals(new Token(WORD, "ugcapi"), tokens.get(22));
    assertEquals(new Token(UNDERSCORE, "_"), tokens.get(23));
    assertEquals(new Token(NUMBER, "1"), tokens.get(24));
    assertEquals(new Token(SPACE, " "), tokens.get(25));
    assertEquals(new Token(WORD, "'a'"), tokens.get(26));
    assertEquals(new Token(SPACE, " "), tokens.get(27));
    assertEquals(new Token(WORD, "'a"), tokens.get(28));
    assertEquals(new Token(SPACE, " "), tokens.get(29));
    assertEquals(new Token(WORD, "a'"), tokens.get(30));
}
Also used : SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) Token(com.yahoo.prelude.query.parser.Token) Tokenizer(com.yahoo.prelude.query.parser.Tokenizer) Test(org.junit.Test)

Example 10 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class TokenizerTestCase method testExactMatchTokenizationEndsByColon.

@Test
public void testExactMatchTokenizationEndsByColon() {
    Index index1 = new Index("testexact1");
    index1.setExact(true, null);
    Index index2 = new Index("testexact2");
    index2.setExact(true, "()/aa*::*&");
    IndexFacts facts = new IndexFacts();
    facts.addIndex("testsd", index1);
    facts.addIndex("testsd", index2);
    Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
    IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet());
    List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:!/%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&b:", session);
    assertEquals(new Token(WORD, "normal"), tokens.get(0));
    assertEquals(new Token(SPACE, " "), tokens.get(1));
    assertEquals(new Token(WORD, "a"), tokens.get(2));
    assertEquals(new Token(COLON, ":"), tokens.get(3));
    assertEquals(new Token(WORD, "b"), tokens.get(4));
    assertEquals(new Token(SPACE, " "), tokens.get(5));
    assertEquals(new Token(LBRACE, "("), tokens.get(6));
    assertEquals(new Token(WORD, "normal"), tokens.get(7));
    assertEquals(new Token(SPACE, " "), tokens.get(8));
    assertEquals(new Token(WORD, "testexact1"), tokens.get(9));
    assertEquals(new Token(COLON, ":"), tokens.get(10));
    assertEquals(new Token(WORD, "!/%#%&+-+"), tokens.get(11));
    assertEquals(new Token(SPACE, " "), tokens.get(12));
    assertEquals(new Token(RBRACE, ")"), tokens.get(13));
    assertEquals(new Token(SPACE, " "), tokens.get(14));
    assertEquals(new Token(WORD, "testexact2"), tokens.get(15));
    assertEquals(new Token(COLON, ":"), tokens.get(16));
    assertEquals(new Token(WORD, "ho_/&%&/"), tokens.get(17));
    assertEquals(new Token(WORD, "b"), tokens.get(18));
    assertEquals(new Token(COLON, ":"), tokens.get(19));
}
Also used : SimpleLinguistics(com.yahoo.language.simple.SimpleLinguistics) IndexFacts(com.yahoo.prelude.IndexFacts) Index(com.yahoo.prelude.Index) Token(com.yahoo.prelude.query.parser.Token) Tokenizer(com.yahoo.prelude.query.parser.Tokenizer) Test(org.junit.Test)

Aggregations

SimpleLinguistics (com.yahoo.language.simple.SimpleLinguistics)42 Test (org.junit.Test)37 Token (com.yahoo.prelude.query.parser.Token)17 Tokenizer (com.yahoo.prelude.query.parser.Tokenizer)17 Linguistics (com.yahoo.language.Linguistics)10 Index (com.yahoo.prelude.Index)7 IndexFacts (com.yahoo.prelude.IndexFacts)7 StringFieldValue (com.yahoo.document.datatypes.StringFieldValue)6 AnnotatorConfig (com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig)5 SpecialTokenRegistry (com.yahoo.prelude.query.parser.SpecialTokenRegistry)3 Query (com.yahoo.search.Query)3 Execution (com.yahoo.search.searchchain.Execution)3 SimpleTestAdapter (com.yahoo.vespa.indexinglanguage.SimpleTestAdapter)3 InputExpression (com.yahoo.vespa.indexinglanguage.expressions.InputExpression)3 Pair (com.yahoo.collections.Pair)2 FieldValue (com.yahoo.document.datatypes.FieldValue)2 IntegerFieldValue (com.yahoo.document.datatypes.IntegerFieldValue)2 RendererRegistry (com.yahoo.search.rendering.RendererRegistry)2 ArithmeticExpression (com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression)2 AttributeExpression (com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression)2