use of com.yahoo.prelude.query.parser.SpecialTokenRegistry in project vespa by vespa-engine.
the class ParserEnvironment method fromExecutionContext.
public static ParserEnvironment fromExecutionContext(Execution.Context context) {
ParserEnvironment env = new ParserEnvironment();
if (context == null) {
return env;
}
if (context.getIndexFacts() != null) {
env.setIndexFacts(context.getIndexFacts());
}
if (context.getLinguistics() != null) {
env.setLinguistics(context.getLinguistics());
}
SpecialTokenRegistry registry = context.getTokenRegistry();
if (registry != null) {
env.setSpecialTokens(registry.getSpecialTokens("default"));
}
return env;
}
use of com.yahoo.prelude.query.parser.SpecialTokenRegistry in project vespa by vespa-engine.
the class RewriterFeaturesTestCase method testConvertStringToQTree.
@Test
public final void testConvertStringToQTree() {
Execution placeholder = new Execution(Context.createContextStub());
SpecialTokenRegistry tokenRegistry = new SpecialTokenRegistry(new SpecialtokensConfig(new SpecialtokensConfig.Builder().tokenlist(new Tokenlist.Builder().name("default").tokens(new Tokens.Builder().token(ASCII_ELLIPSIS)))));
placeholder.context().setTokenRegistry(tokenRegistry);
Query query = new Query();
query.getModel().setExecution(placeholder);
Item parsed = RewriterFeatures.convertStringToQTree(query, "a b c " + ASCII_ELLIPSIS);
assertSame(AndItem.class, parsed.getClass());
assertEquals(4, ((CompositeItem) parsed).getItemCount());
assertEquals(ASCII_ELLIPSIS, ((CompositeItem) parsed).getItem(3).toString());
}
use of com.yahoo.prelude.query.parser.SpecialTokenRegistry in project vespa by vespa-engine.
the class TokenizerTestCase method testSpecialTokenConfigurationOther.
@Test
public void testSpecialTokenConfigurationOther() {
String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg";
SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile);
assertEquals("Special tokens configured", 6, r.getSpecialTokens("default").size());
assertEquals("Special tokens configured", 4, r.getSpecialTokens("other").size());
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(r.getSpecialTokens("other"));
List<?> tokens = tokenizer.tokenize("with space,!!!*** [huh] or ------ " + "know, &&&%%% b.s.d.");
assertEquals(new Token(WORD, "with"), tokens.get(0));
assertEquals(new Token(SPACE, " "), tokens.get(1));
assertEquals(new Token(WORD, "space"), tokens.get(2));
assertEquals(new Token(COMMA, ","), tokens.get(3));
assertEquals(new Token(WORD, "!!!***"), tokens.get(4));
assertEquals(new Token(SPACE, " "), tokens.get(5));
assertEquals(new Token(WORD, "[huh]"), tokens.get(6));
assertEquals(new Token(SPACE, " "), tokens.get(7));
assertEquals(new Token(WORD, "or"), tokens.get(8));
assertEquals(new Token(SPACE, " "), tokens.get(9));
assertEquals(new Token(WORD, "------"), tokens.get(10));
assertEquals(new Token(SPACE, " "), tokens.get(11));
assertEquals(new Token(WORD, "know"), tokens.get(12));
assertEquals(new Token(COMMA, ","), tokens.get(13));
assertEquals(new Token(SPACE, " "), tokens.get(14));
assertEquals(new Token(WORD, "&&&%%%"), tokens.get(15));
assertEquals(new Token(SPACE, " "), tokens.get(16));
assertEquals(new Token(WORD, "b"), tokens.get(17));
assertEquals(new Token(DOT, "."), tokens.get(18));
assertEquals(new Token(WORD, "s"), tokens.get(19));
assertEquals(new Token(DOT, "."), tokens.get(20));
assertEquals(new Token(WORD, "d"), tokens.get(21));
assertEquals(new Token(DOT, "."), tokens.get(22));
assertTrue(((Token) tokens.get(10)).isSpecial());
}
use of com.yahoo.prelude.query.parser.SpecialTokenRegistry in project vespa by vespa-engine.
the class TokenizerTestCase method testSpecialTokenConfigurationDefault.
@Test
public void testSpecialTokenConfigurationDefault() {
String tokenFile = "file:src/test/java/com/yahoo/prelude/query/parser/test/specialtokens.cfg";
SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile);
assertEquals("Special tokens configured", 6, r.getSpecialTokens("default").size());
assertEquals("Special tokens configured", 4, r.getSpecialTokens("other").size());
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(r.getSpecialTokens("default"));
List<?> tokens = tokenizer.tokenize("with space, c++ or .... know, not b.s.d.");
assertEquals(new Token(WORD, "with space"), tokens.get(0));
assertEquals(new Token(COMMA, ","), tokens.get(1));
assertEquals(new Token(SPACE, " "), tokens.get(2));
assertEquals(new Token(WORD, "c++"), tokens.get(3));
assertEquals(new Token(SPACE, " "), tokens.get(4));
assertEquals(new Token(WORD, "or"), tokens.get(5));
assertEquals(new Token(SPACE, " "), tokens.get(6));
assertEquals(new Token(WORD, "...."), tokens.get(7));
assertEquals(new Token(SPACE, " "), tokens.get(8));
assertEquals(new Token(WORD, "know"), tokens.get(9));
assertEquals(new Token(COMMA, ","), tokens.get(10));
assertEquals(new Token(SPACE, " "), tokens.get(11));
assertEquals(new Token(WORD, "not"), tokens.get(12));
assertEquals(new Token(SPACE, " "), tokens.get(13));
assertEquals(new Token(WORD, "b.s.d."), tokens.get(14));
}
use of com.yahoo.prelude.query.parser.SpecialTokenRegistry in project vespa by vespa-engine.
the class TokenizerTestCase method testSpecialTokenConfigurationMissing.
@Test
public void testSpecialTokenConfigurationMissing() {
String tokenFile = "file:source/bogus/specialtokens.cfg";
SpecialTokenRegistry r = new SpecialTokenRegistry(tokenFile);
Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
tokenizer.setSpecialTokens(r.getSpecialTokens("other"));
List<?> tokens = tokenizer.tokenize("c++");
assertEquals(new Token(WORD, "c"), tokens.get(0));
assertEquals(new Token(PLUS, "+"), tokens.get(1));
assertEquals(new Token(PLUS, "+"), tokens.get(2));
}
Aggregations