Examples with Language com.yahoo.language.Language used on opensource projects

Example 1 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class NormalizingSearcher method normalizeBody.

private Query normalizeBody(Query query, IndexFacts.Session indexFacts) {
    Item root = query.getModel().getQueryTree().getRoot();
    Language language = query.getModel().getParsingLanguage();
    if (root instanceof BlockItem) {
        List<Item> rootItems = new ArrayList<>(1);
        rootItems.add(root);
        ListIterator<Item> i = rootItems.listIterator();
        i.next();
        normalizeBlocks(language, indexFacts, (BlockItem) root, i);
        if (// give up normalizing if the root was removed
        !rootItems.isEmpty())
            query.getModel().getQueryTree().setRoot(rootItems.get(0));
    } else if (root instanceof CompositeItem) {
        query.getModel().getQueryTree().setRoot(normalizeComposite(language, indexFacts, (CompositeItem) root));
    }
    return query;
}

Example 2 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class CJKSearcher method search.

@Override
public Result search(Query query, Execution execution) {
    Language language = query.getModel().getParsingLanguage();
    if (!language.isCjk())
        return execution.search(query);
    QueryTree tree = query.getModel().getQueryTree();
    tree.setRoot(transform(tree.getRoot()));
    query.trace("Rewriting for CJK behavior for implicit phrases", true, 2);
    return execution.search(query);
}

Example 3 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class AnnotatorConfigTestCase method requireThatAccessorsWork.

@Test
public void requireThatAccessorsWork() {
    AnnotatorConfig config = new AnnotatorConfig();
    for (Language language : Language.values()) {
        config.setLanguage(language);
        assertEquals(language, config.getLanguage());
    }
    for (StemMode mode : StemMode.values()) {
        config.setStemMode(mode);
        assertEquals(mode, config.getStemMode());
    }
    config.setRemoveAccents(true);
    assertTrue(config.getRemoveAccents());
    config.setRemoveAccents(false);
    assertFalse(config.getRemoveAccents());
}

Example 4 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class Model method getParsingLanguage.

/**
 * Gets the language to use for parsing. If this is explicitly set in the model, that language is returned.
 * Otherwise, if a query tree is already produced and any node in it specifies a language the first such
 * node encountered in a depth first
 * left to right search is returned. Otherwise the language is guessed from the query string.
 * If this does not yield an actual language, English is returned as the default.
 *
 * @return the language determined, never null
 */
// TODO: We can support multiple languages per query by changing searchers which call this
// to look up the query to use at each point from item.getLanguage
// with this as fallback for query branches where no parent item specifies language
public Language getParsingLanguage(String languageDetectionText) {
    Language language = getLanguage();
    if (language != null)
        return language;
    language = Language.fromEncoding(encoding);
    if (language != Language.UNKNOWN)
        return language;
    if (queryTree != null)
        language = languageBelow(queryTree);
    if (language != Language.UNKNOWN)
        return language;
    Linguistics linguistics = execution.context().getLinguistics();
    if (linguistics != null)
        // TODO: Set language if detected
        language = linguistics.getDetector().detect(languageDetectionText, null).getLanguage();
    if (language != Language.UNKNOWN)
        return language;
    return Language.ENGLISH;
}

Example 5 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class Model method traceLanguage.

/**
 * Creates trace a message of language detection results into this Model
 * instance's parent query. Do note this will give bogus results if the
 * Execution instance is not set correctly. This is done automatically
 * inside {@link Execution#search(Query)}. If tracing the same place as
 * creating the query instance, {@link #setExecution(Execution)} has to be
 * invoked first with the same Execution instance the query is intended to
 * be run by.
 *
 * @deprecated do not use; language can now be assigned later and for parts of the query tree, making this quite useless
 */
@Deprecated
public void traceLanguage() {
    if (getParent().getTraceLevel() < 2)
        return;
    if (language != null) {
        getParent().trace("Language " + getLanguage() + " specified directly as a parameter", false, 2);
    } else {
        Language l = getParsingLanguage();
        // Don't include the query, it will trigger query parsing
        getParent().trace("Detected language: " + l, false, 2);
        getParent().trace("Language " + l + " determined by " + (Language.fromEncoding(encoding) != Language.UNKNOWN ? "query encoding" : "the characters in the terms") + ".", false, 2);
    }
}