Examples with Tokens com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens used on opensource projects

Example 1 with Tokens

use of com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens in project vespa by vespa-engine.

the class SpecialTokenRegistry method build.

private void build(SpecialtokensConfig config) {
    List<SpecialTokens> list = new ArrayList<>();
    for (Iterator<Tokenlist> i = config.tokenlist().iterator(); i.hasNext(); ) {
        Tokenlist tokenList = i.next();
        SpecialTokens tokens = new SpecialTokens(tokenList.name());
        for (Iterator<Tokens> j = tokenList.tokens().iterator(); j.hasNext(); ) {
            Tokens token = j.next();
            tokens.addSpecialToken(token.token(), token.replace());
        }
        tokens.freeze();
        list.add(tokens);
    }
    addSpecialTokens(list);
}

Example 2 with Tokens

use of com.yahoo.vespa.configdefinition.SpecialtokensConfig.Tokenlist.Tokens in project vespa by vespa-engine.

the class RewriterFeaturesTestCase method testConvertStringToQTree.

@Test
public final void testConvertStringToQTree() {
    Execution placeholder = new Execution(Context.createContextStub());
    SpecialTokenRegistry tokenRegistry = new SpecialTokenRegistry(new SpecialtokensConfig(new SpecialtokensConfig.Builder().tokenlist(new Tokenlist.Builder().name("default").tokens(new Tokens.Builder().token(ASCII_ELLIPSIS)))));
    placeholder.context().setTokenRegistry(tokenRegistry);
    Query query = new Query();
    query.getModel().setExecution(placeholder);
    Item parsed = RewriterFeatures.convertStringToQTree(query, "a b c " + ASCII_ELLIPSIS);
    assertSame(AndItem.class, parsed.getClass());
    assertEquals(4, ((CompositeItem) parsed).getItemCount());
    assertEquals(ASCII_ELLIPSIS, ((CompositeItem) parsed).getItem(3).toString());
}