Examples with MockTokenizer org.apache.lucene.analysis.MockTokenizer used on opensource projects

Example 16 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestFingerprintFilter method testDupsAndSorting.

public void testDupsAndSorting() throws Exception {
    for (final boolean consumeAll : new boolean[] { true, false }) {
        MockTokenizer tokenizer = whitespaceMockTokenizer("B A B E");
        tokenizer.setEnableChecks(consumeAll);
        TokenStream stream = new FingerprintFilter(tokenizer);
        assertTokenStreamContents(stream, new String[] { "A B E" });
    }
}

Example 17 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestFingerprintFilter method testSingleToken.

public void testSingleToken() throws Exception {
    for (final boolean consumeAll : new boolean[] { true, false }) {
        MockTokenizer tokenizer = whitespaceMockTokenizer("A1");
        tokenizer.setEnableChecks(consumeAll);
        TokenStream stream = new FingerprintFilter(tokenizer);
        assertTokenStreamContents(stream, new String[] { "A1" });
    }
}

Example 18 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymGraphFilter method getFlattenAnalyzer.

/** Appends FlattenGraphFilter too */
private Analyzer getFlattenAnalyzer(SynonymMap.Builder b, boolean ignoreCase) throws IOException {
    final SynonymMap map = b.build();
    return new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
            // Make a local variable so testRandomHuge doesn't share it across threads!
            SynonymGraphFilter synFilter = new SynonymGraphFilter(tokenizer, map, ignoreCase);
            FlattenGraphFilter flattenFilter = new FlattenGraphFilter(synFilter);
            TestSynonymGraphFilter.this.synFilter = synFilter;
            TestSynonymGraphFilter.this.flattenFilter = flattenFilter;
            return new TokenStreamComponents(tokenizer, flattenFilter);
        }
    };
}

Example 19 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymGraphFilter method testRandomGraphAfter.

// Adds MockGraphTokenFilter after SynFilter:
public void testRandomGraphAfter() throws Exception {
    final int numIters = atLeast(3);
    for (int i = 0; i < numIters; i++) {
        SynonymMap.Builder b = new SynonymMap.Builder(random().nextBoolean());
        final int numEntries = atLeast(10);
        for (int j = 0; j < numEntries; j++) {
            add(b, randomNonEmptyString(), randomNonEmptyString(), random().nextBoolean());
        }
        final SynonymMap map = b.build();
        final boolean ignoreCase = random().nextBoolean();
        final boolean doFlatten = random().nextBoolean();
        final Analyzer analyzer = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
                TokenStream syns = new SynonymGraphFilter(tokenizer, map, ignoreCase);
                TokenStream graph = new MockGraphTokenFilter(random(), syns);
                if (doFlatten) {
                    graph = new FlattenGraphFilter(graph);
                }
                return new TokenStreamComponents(tokenizer, graph);
            }
        };
        checkRandomData(random(), analyzer, 100);
        analyzer.close();
    }
}

Example 20 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymGraphFilter method getAnalyzer.

private Analyzer getAnalyzer(SynonymMap.Builder b, final boolean ignoreCase) throws IOException {
    final SynonymMap map = b.build();
    return new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            // Make a local variable so testRandomHuge doesn't share it across threads!
            SynonymGraphFilter synFilter = new SynonymGraphFilter(tokenizer, map, ignoreCase);
            TestSynonymGraphFilter.this.flattenFilter = null;
            TestSynonymGraphFilter.this.synFilter = synFilter;
            return new TokenStreamComponents(tokenizer, synFilter);
        }
    };
}