Search in sources :

Example 76 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymGraphFilter method getAnalyzer.

private Analyzer getAnalyzer(SynonymMap.Builder b, final boolean ignoreCase) throws IOException {
    final SynonymMap map = b.build();
    return new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            // Make a local variable so testRandomHuge doesn't share it across threads!
            SynonymGraphFilter synFilter = new SynonymGraphFilter(tokenizer, map, ignoreCase);
            TestSynonymGraphFilter.this.flattenFilter = null;
            TestSynonymGraphFilter.this.synFilter = synFilter;
            return new TokenStreamComponents(tokenizer, synFilter);
        }
    };
}
Also used : MockTokenizer(org.apache.lucene.analysis.MockTokenizer) Analyzer(org.apache.lucene.analysis.Analyzer) MockAnalyzer(org.apache.lucene.analysis.MockAnalyzer) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer)

Example 77 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymGraphFilter method testZeroSyns.

public void testZeroSyns() throws Exception {
    Tokenizer tokenizer = new MockTokenizer();
    tokenizer.setReader(new StringReader("aa bb"));
    try {
        new SynonymGraphFilter(tokenizer, new SynonymMap.Builder(true).build(), true);
        fail("did not hit expected exception");
    } catch (IllegalArgumentException iae) {
        // expected
        assertEquals("fst must be non-null", iae.getMessage());
    }
}
Also used : MockTokenizer(org.apache.lucene.analysis.MockTokenizer) CharsRefBuilder(org.apache.lucene.util.CharsRefBuilder) IntsRefBuilder(org.apache.lucene.util.IntsRefBuilder) StringReader(java.io.StringReader) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer)

Example 78 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestApostropheFilterFactory method testApostrophes.

/**
   * Ensure the filter actually removes characters after an apostrophe.
   */
public void testApostrophes() throws Exception {
    Reader reader = new StringReader("Türkiye'de 2003'te Van Gölü'nü gördüm");
    TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    ((Tokenizer) stream).setReader(reader);
    stream = tokenFilterFactory("Apostrophe").create(stream);
    assertTokenStreamContents(stream, new String[] { "Türkiye", "2003", "Van", "Gölü", "gördüm" });
}
Also used : MockTokenizer(org.apache.lucene.analysis.MockTokenizer) TokenStream(org.apache.lucene.analysis.TokenStream) StringReader(java.io.StringReader) StringReader(java.io.StringReader) Reader(java.io.Reader) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer)

Example 79 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymMapFilter method testRecursion.

public void testRecursion() throws Exception {
    b = new SynonymMap.Builder(true);
    final boolean keepOrig = false;
    add("zoo", "zoo", keepOrig);
    final SynonymMap map = b.build();
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
            return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
        }
    };
    assertAnalyzesTo(a, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "$", "zoo" }, new int[] { 1, 1, 1, 1 });
    a.close();
}
Also used : MockTokenizer(org.apache.lucene.analysis.MockTokenizer) Analyzer(org.apache.lucene.analysis.Analyzer) MockAnalyzer(org.apache.lucene.analysis.MockAnalyzer) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer) KeywordTokenizer(org.apache.lucene.analysis.core.KeywordTokenizer)

Example 80 with MockTokenizer

use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.

the class TestSynonymMapFilter method testDontKeepOrig.

public void testDontKeepOrig() throws Exception {
    b = new SynonymMap.Builder(true);
    add("a b", "foo", false);
    final SynonymMap map = b.build();
    final Analyzer analyzer = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
            return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
        }
    };
    assertAnalyzesTo(analyzer, "a b c", new String[] { "foo", "c" }, new int[] { 0, 4 }, new int[] { 3, 5 }, null, new int[] { 1, 1 }, new int[] { 1, 1 }, true);
    checkAnalysisConsistency(random(), analyzer, false, "a b c");
    analyzer.close();
}
Also used : MockTokenizer(org.apache.lucene.analysis.MockTokenizer) Analyzer(org.apache.lucene.analysis.Analyzer) MockAnalyzer(org.apache.lucene.analysis.MockAnalyzer) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer) KeywordTokenizer(org.apache.lucene.analysis.core.KeywordTokenizer)

Aggregations

MockTokenizer (org.apache.lucene.analysis.MockTokenizer)280 Tokenizer (org.apache.lucene.analysis.Tokenizer)204 Analyzer (org.apache.lucene.analysis.Analyzer)161 StringReader (java.io.StringReader)118 TokenStream (org.apache.lucene.analysis.TokenStream)116 KeywordTokenizer (org.apache.lucene.analysis.core.KeywordTokenizer)106 Reader (java.io.Reader)59 MockAnalyzer (org.apache.lucene.analysis.MockAnalyzer)54 CharArraySet (org.apache.lucene.analysis.CharArraySet)44 Directory (org.apache.lucene.store.Directory)36 Document (org.apache.lucene.document.Document)31 BytesRef (org.apache.lucene.util.BytesRef)25 SetKeywordMarkerFilter (org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter)21 TextField (org.apache.lucene.document.TextField)20 CannedTokenStream (org.apache.lucene.analysis.CannedTokenStream)18 Field (org.apache.lucene.document.Field)17 FieldType (org.apache.lucene.document.FieldType)14 StringField (org.apache.lucene.document.StringField)11 Input (org.apache.lucene.search.suggest.Input)11 InputArrayIterator (org.apache.lucene.search.suggest.InputArrayIterator)11