Examples with Analyzer org.apache.lucene.analysis.Analyzer used on opensource projects

Example 71 with Analyzer

use of org.apache.lucene.analysis.Analyzer in project jackrabbit-oak by apache.

the class LucenePropertyIndex method getLuceneRequest.

/**
     * Get the Lucene query for the given filter.
     *
     * @param plan index plan containing filter details
     * @param reader the Lucene reader
     * @return the Lucene query
     */
private static LuceneRequestFacade getLuceneRequest(IndexPlan plan, IndexAugmentorFactory augmentorFactory, IndexReader reader) {
    FulltextQueryTermsProvider augmentor = getIndexAgumentor(plan, augmentorFactory);
    List<Query> qs = new ArrayList<Query>();
    Filter filter = plan.getFilter();
    FullTextExpression ft = filter.getFullTextConstraint();
    PlanResult planResult = getPlanResult(plan);
    IndexDefinition defn = planResult.indexDefinition;
    Analyzer analyzer = defn.getAnalyzer();
    if (ft == null) {
    // there might be no full-text constraint
    // when using the LowCostLuceneIndexProvider
    // which is used for testing
    } else {
        qs.add(getFullTextQuery(plan, ft, analyzer, augmentor));
    }
    //Check if native function is supported
    PropertyRestriction pr = null;
    if (defn.hasFunctionDefined()) {
        pr = filter.getPropertyRestriction(defn.getFunctionName());
    }
    if (pr != null) {
        String query = String.valueOf(pr.first.getValue(pr.first.getType()));
        QueryParser queryParser = new QueryParser(VERSION, "", analyzer);
        if (query.startsWith("mlt?")) {
            String mltQueryString = query.replace("mlt?", "");
            if (reader != null) {
                Query moreLikeThis = MoreLikeThisHelper.getMoreLikeThis(reader, analyzer, mltQueryString);
                if (moreLikeThis != null) {
                    qs.add(moreLikeThis);
                }
            }
        } else if (query.startsWith("spellcheck?")) {
            String spellcheckQueryString = query.replace("spellcheck?", "");
            if (reader != null) {
                return new LuceneRequestFacade<SpellcheckHelper.SpellcheckQuery>(SpellcheckHelper.getSpellcheckQuery(spellcheckQueryString, reader));
            }
        } else if (query.startsWith("suggest?")) {
            String suggestQueryString = query.replace("suggest?", "");
            if (reader != null) {
                return new LuceneRequestFacade<SuggestHelper.SuggestQuery>(SuggestHelper.getSuggestQuery(suggestQueryString));
            }
        } else {
            try {
                qs.add(queryParser.parse(query));
            } catch (ParseException e) {
                throw new RuntimeException(e);
            }
        }
    } else if (planResult.evaluateNonFullTextConstraints()) {
        addNonFullTextConstraints(qs, plan, reader);
    }
    if (qs.size() == 0 && plan.getSortOrder() != null) {
        //This case indicates that query just had order by and no
        //property restriction defined. In this case property
        //existence queries for each sort entry
        List<OrderEntry> orders = removeNativeSort(plan.getSortOrder());
        for (int i = 0; i < orders.size(); i++) {
            OrderEntry oe = orders.get(i);
            PropertyDefinition pd = planResult.getOrderedProperty(i);
            PropertyRestriction orderRest = new PropertyRestriction();
            orderRest.propertyName = oe.getPropertyName();
            Query q = createQuery(orderRest, pd);
            if (q != null) {
                qs.add(q);
            }
        }
    }
    if (qs.size() == 0) {
        if (reader == null) {
            //just return match all queries
            return new LuceneRequestFacade<Query>(new MatchAllDocsQuery());
        }
        //be returned (if the index definition has a single rule)
        if (planResult.evaluateNodeTypeRestriction()) {
            return new LuceneRequestFacade<Query>(new MatchAllDocsQuery());
        }
        throw new IllegalStateException("No query created for filter " + filter);
    }
    return performAdditionalWraps(qs);
}

Example 72 with Analyzer

use of org.apache.lucene.analysis.Analyzer in project lucene-solr by apache.

the class SynonymFilterFactory method inform.

@Override
public void inform(ResourceLoader loader) throws IOException {
    final TokenizerFactory factory = tokenizerFactory == null ? null : loadTokenizerFactory(loader, tokenizerFactory);
    Analyzer analyzer;
    if (analyzerName != null) {
        analyzer = loadAnalyzer(loader, analyzerName);
    } else {
        analyzer = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer() : factory.create();
                TokenStream stream = ignoreCase ? new LowerCaseFilter(tokenizer) : tokenizer;
                return new TokenStreamComponents(tokenizer, stream);
            }
        };
    }
    try (Analyzer a = analyzer) {
        String formatClass = format;
        if (format == null || format.equals("solr")) {
            formatClass = SolrSynonymParser.class.getName();
        } else if (format.equals("wordnet")) {
            formatClass = WordnetSynonymParser.class.getName();
        }
        // TODO: expose dedup as a parameter?
        map = loadSynonyms(loader, formatClass, true, a);
    } catch (ParseException e) {
        throw new IOException("Error parsing synonyms file:", e);
    }
}

Example 73 with Analyzer

use of org.apache.lucene.analysis.Analyzer in project lucene-solr by apache.

the class SynonymGraphFilterFactory method inform.

@Override
public void inform(ResourceLoader loader) throws IOException {
    final TokenizerFactory factory = tokenizerFactory == null ? null : loadTokenizerFactory(loader, tokenizerFactory);
    Analyzer analyzer;
    if (analyzerName != null) {
        analyzer = loadAnalyzer(loader, analyzerName);
    } else {
        analyzer = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer() : factory.create();
                TokenStream stream = ignoreCase ? new LowerCaseFilter(tokenizer) : tokenizer;
                return new TokenStreamComponents(tokenizer, stream);
            }
        };
    }
    try (Analyzer a = analyzer) {
        String formatClass = format;
        if (format == null || format.equals("solr")) {
            formatClass = SolrSynonymParser.class.getName();
        } else if (format.equals("wordnet")) {
            formatClass = WordnetSynonymParser.class.getName();
        }
        // TODO: expose dedup as a parameter?
        map = loadSynonyms(loader, formatClass, true, a);
    } catch (ParseException e) {
        throw new IOException("Error parsing synonyms file:", e);
    }
}

Example 74 with Analyzer

use of org.apache.lucene.analysis.Analyzer in project lucene-solr by apache.

the class TestCatalanAnalyzer method testBasics.

/** test stopwords and stemming */
public void testBasics() throws IOException {
    Analyzer a = new CatalanAnalyzer();
    // stemming
    checkOneTerm(a, "llengües", "llengu");
    checkOneTerm(a, "llengua", "llengu");
    // stopword
    assertAnalyzesTo(a, "un", new String[] {});
    a.close();
}

Example 75 with Analyzer

use of org.apache.lucene.analysis.Analyzer in project lucene-solr by apache.

the class TestCatalanAnalyzer method testExclude.

/** test use of exclusion set */
public void testExclude() throws IOException {
    CharArraySet exclusionSet = new CharArraySet(asSet("llengües"), false);
    Analyzer a = new CatalanAnalyzer(CatalanAnalyzer.getDefaultStopSet(), exclusionSet);
    checkOneTerm(a, "llengües", "llengües");
    checkOneTerm(a, "llengua", "llengu");
    a.close();
}