Examples with Index com.yahoo.prelude.Index used on opensource projects

Example 21 with Index

use of com.yahoo.prelude.Index in project vespa by vespa-engine.

the class NormalizingSearcher method normalizeWord.

private void normalizeWord(Language language, IndexFacts.Session indexFacts, TermItem term, ListIterator<Item> i) {
    if (!(term instanceof WordItem))
        return;
    if (!term.isNormalizable())
        return;
    Index index = indexFacts.getIndex(term.getIndexName());
    if (index.isAttribute())
        return;
    if (!index.getNormalize())
        return;
    WordItem word = (WordItem) term;
    String accentDropped = linguistics.getTransformer().accentDrop(word.getWord(), language);
    if (accentDropped.length() == 0)
        i.remove();
    else
        word.setWord(accentDropped);
}

Example 22 with Index

use of com.yahoo.prelude.Index in project vespa by vespa-engine.

the class NGramSearcher method recombineNGrams.

private void recombineNGrams(Iterator<Hit> hits, IndexFacts.Session session) {
    while (hits.hasNext()) {
        Hit hit = hits.next();
        if (hit.isMeta())
            continue;
        Object sddocname = hit.getField(Hit.SDDOCNAME_FIELD);
        if (sddocname == null)
            return;
        for (String fieldName : hit.fieldKeys()) {
            Index index = session.getIndex(fieldName, sddocname.toString());
            if (index.isNGram() && (index.getHighlightSummary() || index.getDynamicSummary())) {
                hit.setField(fieldName, recombineNGramsField(hit.getField(fieldName), index.getGramSize()));
            }
        }
    }
}

Example 23 with Index

use of com.yahoo.prelude.Index in project vespa by vespa-engine.

the class NGramSearcher method rewriteToNGramMatching.

private boolean rewriteToNGramMatching(Item item, int indexInParent, IndexFacts.Session indexFacts, Query query) {
    boolean rewritten = false;
    if (item instanceof SegmentItem) {
        // handle CJK segmented terms which should be grams instead
        SegmentItem segments = (SegmentItem) item;
        Index index = indexFacts.getIndex(segments.getIndexName());
        if (index.isNGram()) {
            Item grams = splitToGrams(segments, toLowerCase(segments.getRawWord()), index.getGramSize(), query);
            replaceItemByGrams(item, grams, indexInParent);
            rewritten = true;
        }
    } else if (item instanceof CompositeItem) {
        CompositeItem composite = (CompositeItem) item;
        for (int i = 0; i < composite.getItemCount(); i++) rewritten = rewriteToNGramMatching(composite.getItem(i), i, indexFacts, query) || rewritten;
    } else if (item instanceof TermItem) {
        TermItem term = (TermItem) item;
        Index index = indexFacts.getIndex(term.getIndexName());
        if (index.isNGram()) {
            Item grams = splitToGrams(term, term.stringValue(), index.getGramSize(), query);
            replaceItemByGrams(item, grams, indexInParent);
            rewritten = true;
        }
    }
    return rewritten;
}

Example 24 with Index

use of com.yahoo.prelude.Index in project vespa by vespa-engine.

the class QueryCombinator method addAndItems.

private void addAndItems(Query query, Iterable<QueryPart> pieces, Execution.Context context) {
    IndexFacts indexFacts = context.getIndexFacts();
    IndexFacts.Session session = indexFacts.newSession(query);
    Set<String> usedSources = new HashSet<>(session.documentTypes());
    Language language = query.getModel().getParsingLanguage();
    for (QueryPart part : pieces) {
        String defaultIndex;
        Item item = null;
        Index index = session.getIndex(part.defaultIndex);
        if (index == Index.nullIndex) {
            defaultIndex = null;
        } else {
            defaultIndex = part.defaultIndex;
        }
        try {
            CustomParser parser = (CustomParser) ParserFactory.newInstance(query.getModel().getType(), ParserEnvironment.fromExecutionContext(context));
            item = parser.parse(part.query, null, language, usedSources, indexFacts, defaultIndex);
        } catch (RuntimeException e) {
            String err = Exceptions.toMessageString(e);
            query.trace("Query parser threw an exception: " + err, true, 1);
            getLogger().log(LogLevel.WARNING, "Query parser threw exception searcher QueryCombinator for " + query.getHttpRequest().toString() + ", query part " + part.query + ": " + err);
        }
        if (item == null) {
            continue;
        }
        if (defaultIndex == null) {
            assignDefaultIndex(item, part.defaultIndex);
        }
        addAndItem(query.getModel().getQueryTree(), item);
    }
}

Example 25 with Index

use of com.yahoo.prelude.Index in project vespa by vespa-engine.

the class ValidatePredicateSearcherTestCase method doSearch.

private static Result doSearch(ValidatePredicateSearcher searcher, String yqlQuery, String command) {
    QueryTree queryTree = new YqlParser(new ParserEnvironment()).parse(new Parsable().setQuery(yqlQuery));
    Query query = new Query();
    query.getModel().getQueryTree().setRoot(queryTree.getRoot());
    TreeMap<String, List<String>> masterClusters = new TreeMap<>();
    masterClusters.put("cluster", Arrays.asList("document"));
    SearchDefinition searchDefinition = new SearchDefinition("document");
    Index index = new Index("predicate_field");
    index.addCommand(command);
    searchDefinition.addIndex(index);
    Map<String, SearchDefinition> searchDefinitionMap = new HashMap<>();
    searchDefinitionMap.put("document", searchDefinition);
    IndexFacts indexFacts = new IndexFacts(new IndexModel(masterClusters, searchDefinitionMap, searchDefinition));
    Execution.Context context = new Execution.Context(null, indexFacts, null, new RendererRegistry(MoreExecutors.directExecutor()), new SimpleLinguistics());
    return new Execution(searcher, context).search(query);
}