Examples with FlattenGraphFilter org.apache.lucene.analysis.core.FlattenGraphFilter used on opensource projects

Example 1 with FlattenGraphFilter

use of org.apache.lucene.analysis.core.FlattenGraphFilter in project lucene-solr by apache.

the class TestSynonymGraphFilter method getFlattenAnalyzer.

/** Appends FlattenGraphFilter too */
private Analyzer getFlattenAnalyzer(SynonymMap.Builder b, boolean ignoreCase) throws IOException {
    final SynonymMap map = b.build();
    return new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
            // Make a local variable so testRandomHuge doesn't share it across threads!
            SynonymGraphFilter synFilter = new SynonymGraphFilter(tokenizer, map, ignoreCase);
            FlattenGraphFilter flattenFilter = new FlattenGraphFilter(synFilter);
            TestSynonymGraphFilter.this.synFilter = synFilter;
            TestSynonymGraphFilter.this.flattenFilter = flattenFilter;
            return new TokenStreamComponents(tokenizer, flattenFilter);
        }
    };
}

Example 2 with FlattenGraphFilter

use of org.apache.lucene.analysis.core.FlattenGraphFilter in project lucene-solr by apache.

the class TestSynonymGraphFilter method testRandomGraphAfter.

// Adds MockGraphTokenFilter after SynFilter:
public void testRandomGraphAfter() throws Exception {
    final int numIters = atLeast(3);
    for (int i = 0; i < numIters; i++) {
        SynonymMap.Builder b = new SynonymMap.Builder(random().nextBoolean());
        final int numEntries = atLeast(10);
        for (int j = 0; j < numEntries; j++) {
            add(b, randomNonEmptyString(), randomNonEmptyString(), random().nextBoolean());
        }
        final SynonymMap map = b.build();
        final boolean ignoreCase = random().nextBoolean();
        final boolean doFlatten = random().nextBoolean();
        final Analyzer analyzer = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
                TokenStream syns = new SynonymGraphFilter(tokenizer, map, ignoreCase);
                TokenStream graph = new MockGraphTokenFilter(random(), syns);
                if (doFlatten) {
                    graph = new FlattenGraphFilter(graph);
                }
                return new TokenStreamComponents(tokenizer, graph);
            }
        };
        checkRandomData(random(), analyzer, 100);
        analyzer.close();
    }
}