Examples with Language com.yahoo.language.Language used on opensource projects

Example 6 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class LegacyCombinator method search.

@Override
public Result search(Query query, Execution execution) {
    Set<QueryPart> pieces;
    Set<String> usedSources;
    IndexFacts indexFacts = execution.context().getIndexFacts();
    try {
        pieces = findQuerySnippets(query.properties());
    } catch (IllegalArgumentException e) {
        query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator got invalid parameters: " + e.getMessage()));
        return execution.search(query);
    }
    if (pieces.size() == 0) {
        return execution.search(query);
    }
    IndexFacts.Session session = indexFacts.newSession(query);
    Language language = query.getModel().getParsingLanguage();
    addAndItems(language, query, pieces, session, execution.context());
    addRankItems(language, query, pieces, session, execution.context());
    try {
        addNotItems(language, query, pieces, session, execution.context());
    } catch (IllegalArgumentException e) {
        query.errors().add(ErrorMessage.createInvalidQueryParameter("LegacyCombinator found only excluding terms, no including."));
        return execution.search(query);
    }
    query.trace("Adding extra query parts.", true, 2);
    return execution.search(query);
}

Example 7 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class TokenizeExpression method doExecute.

@Override
protected void doExecute(ExecutionContext context) {
    StringFieldValue input = (StringFieldValue) context.getValue();
    StringFieldValue output = input.clone();
    context.setValue(output);
    AnnotatorConfig cfg = new AnnotatorConfig(config);
    Language lang = context.resolveLanguage(linguistics);
    if (lang != null) {
        cfg.setLanguage(lang);
    }
    LinguisticsAnnotator annotator = new LinguisticsAnnotator(linguistics, cfg);
    annotator.annotate(output);
}

Example 8 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class YqlParser method buildUserInput.

@NonNull
private Item buildUserInput(OperatorNode<ExpressionOperator> ast) {
    // TODO add support for default arguments if property results in nothing
    List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
    String wordData = getStringContents(args.get(0));
    Boolean allowEmpty = getAnnotation(ast, USER_INPUT_ALLOW_EMPTY, Boolean.class, Boolean.FALSE, "flag for allowing NullItem to be returned");
    if (allowEmpty && (wordData == null || wordData.isEmpty()))
        return new NullItem();
    String grammar = getAnnotation(ast, USER_INPUT_GRAMMAR, String.class, Query.Type.ALL.toString(), "grammar for handling user input");
    String defaultIndex = getAnnotation(ast, USER_INPUT_DEFAULT_INDEX, String.class, "default", "default index for user input terms");
    Language language = decideParsingLanguage(ast, wordData);
    Item item;
    if (USER_INPUT_RAW.equals(grammar)) {
        item = instantiateWordItem(defaultIndex, wordData, ast, null, SegmentWhen.NEVER, true, language);
    } else if (USER_INPUT_SEGMENT.equals(grammar)) {
        item = instantiateWordItem(defaultIndex, wordData, ast, null, SegmentWhen.ALWAYS, false, language);
    } else {
        item = parseUserInput(grammar, defaultIndex, wordData, language, allowEmpty);
        propagateUserInputAnnotations(ast, item);
    }
    return item;
}

Example 9 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class QueryCombinator method addAndItems.

private void addAndItems(Query query, Iterable<QueryPart> pieces, Execution.Context context) {
    IndexFacts indexFacts = context.getIndexFacts();
    IndexFacts.Session session = indexFacts.newSession(query);
    Set<String> usedSources = new HashSet<>(session.documentTypes());
    Language language = query.getModel().getParsingLanguage();
    for (QueryPart part : pieces) {
        String defaultIndex;
        Item item = null;
        Index index = session.getIndex(part.defaultIndex);
        if (index == Index.nullIndex) {
            defaultIndex = null;
        } else {
            defaultIndex = part.defaultIndex;
        }
        try {
            CustomParser parser = (CustomParser) ParserFactory.newInstance(query.getModel().getType(), ParserEnvironment.fromExecutionContext(context));
            item = parser.parse(part.query, null, language, usedSources, indexFacts, defaultIndex);
        } catch (RuntimeException e) {
            String err = Exceptions.toMessageString(e);
            query.trace("Query parser threw an exception: " + err, true, 1);
            getLogger().log(LogLevel.WARNING, "Query parser threw exception searcher QueryCombinator for " + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err);
        }
        if (item == null) {
            continue;
        }
        if (defaultIndex == null) {
            assignDefaultIndex(item, part.defaultIndex);
        }
        addAndItem(query.getModel().getQueryTree(), item);
    }
}

Example 10 with Language

use of com.yahoo.language.Language in project vespa by vespa-engine.

the class YqlParser method decideParsingLanguage.

private Language decideParsingLanguage(OperatorNode<ExpressionOperator> ast, String wordData) {
    String languageTag = getAnnotation(ast, USER_INPUT_LANGUAGE, String.class, null, "language setting for segmenting query section");
    Language language = Language.fromLanguageTag(languageTag);
    if (language != Language.UNKNOWN)
        return language;
    Optional<Language> explicitLanguage = currentlyParsing.getExplicitLanguage();
    if (explicitLanguage.isPresent())
        return explicitLanguage.get();
    language = detector.detect(wordData, null).getLanguage();
    if (language != Language.UNKNOWN)
        return language;
    return Language.ENGLISH;
}