Examples with SimpleLinguistics com.yahoo.language.simple.SimpleLinguistics used on opensource projects

Example 16 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class TokenizerTestCase method testSpecialTokenCaseInsensitive.

@Test
public void testSpecialTokenCaseInsensitive() {
    Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics());
    tokenizer.setSpecialTokens(createSpecialTokens());
    List<?> tokens = tokenizer.tokenize("The AS/400 is great");
    assertEquals(new Token(WORD, "The"), tokens.get(0));
    assertEquals(new Token(SPACE, " "), tokens.get(1));
    assertEquals(new Token(WORD, "as/400"), tokens.get(2));
    assertEquals(new Token(SPACE, " "), tokens.get(3));
    assertEquals(new Token(WORD, "is"), tokens.get(4));
    assertEquals(new Token(SPACE, " "), tokens.get(5));
    assertEquals(new Token(WORD, "great"), tokens.get(6));
}

Example 17 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class NGramSearcherTestCase method setUp.

@Override
public void setUp() {
    searcher = new NGramSearcher(new SimpleLinguistics());
    indexFacts = new IndexFacts();
    Index defaultIndex = new Index("default");
    defaultIndex.setNGram(true, 3);
    defaultIndex.setDynamicSummary(true);
    indexFacts.addIndex("default", defaultIndex);
    Index test = new Index("test");
    test.setHighlightSummary(true);
    indexFacts.addIndex("default", test);
    Index gram2 = new Index("gram2");
    gram2.setNGram(true, 2);
    gram2.setDynamicSummary(true);
    indexFacts.addIndex("default", gram2);
    Index gram3 = new Index("gram3");
    gram3.setNGram(true, 3);
    gram3.setHighlightSummary(true);
    indexFacts.addIndex("default", gram3);
    Index gram14 = new Index("gram14");
    gram14.setNGram(true, 14);
    gram14.setDynamicSummary(true);
    indexFacts.addIndex("default", gram14);
}

Example 18 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class ValidatePredicateSearcherTestCase method doSearch.

private static Result doSearch(ValidatePredicateSearcher searcher, String yqlQuery, String command) {
    QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery));
    Query query = new Query();
    query.getModel().getQueryTree().setRoot(queryTree.getRoot());
    TreeMap<String, List<String>> masterClusters = new TreeMap<>();
    masterClusters.put("cluster", Arrays.asList("document"));
    SearchDefinition searchDefinition = new SearchDefinition("document");
    Index index = new Index("predicate_field");
    index.addCommand(command);
    searchDefinition.addIndex(index);
    Map<String, SearchDefinition> searchDefinitionMap = new HashMap<>();
    searchDefinitionMap.put("document", searchDefinition);
    IndexFacts indexFacts = new IndexFacts(new IndexModel(masterClusters, searchDefinitionMap, searchDefinition));
    Execution.Context context = new Execution.Context(null, indexFacts, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics());
    return new Execution(searcher, context).search(query);
}

Example 19 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class TilingTestCase method callSearchAndSetRenderer.

private Result callSearchAndSetRenderer(Chain<Searcher> chain, Query query) {
    Execution.Context context = new Execution.Context(null, null, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics());
    Result result = new Execution(chain, context).search(query);
    result.getTemplating().setRenderer(new SearchRendererAdaptor(new TiledTemplateSet()));
    return result;
}

Example 20 with SimpleLinguistics

use of com.yahoo.language.simple.SimpleLinguistics in project vespa by vespa-engine.

the class QueryTestCase method testSimpleFunctionality.

@Test
public void testSimpleFunctionality() {
    Query q = new Query(QueryTestCase.httpEncode("/sdfsd.html?query=this is a simple query&aParameter"));
    assertEquals("this is a simple query", q.getModel().getQueryString());
    assertNotNull(q.getModel().getQueryTree());
    assertNull(q.getModel().getDefaultIndex());
    assertEquals("", q.properties().get("aParameter"));
    assertNull(q.properties().get("notSetParameter"));
    Query query = q;
    String body = "a bb. ccc??!";
    Linguistics linguistics = new SimpleLinguistics();
    AndItem and = new AndItem();
    for (Token token : linguistics.getTokenizer().tokenize(body, Language.ENGLISH, StemMode.SHORTEST, true)) {
        if (token.isIndexable())
            and.addItem(new WordItem(token.getTokenString(), "body"));
    }
    query.getModel().getQueryTree().setRoot(and);
    System.out.println(query);
}