Examples with Tokenizer org.apache.lucene.analysis.Tokenizer used on opensource projects

Example 56 with Tokenizer

use of org.apache.lucene.analysis.Tokenizer in project lucene-solr by apache.

the class TestIndonesianStemmer method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    /* full stemming, no stopwords */
    a = new Analyzer() {

        @Override
        public TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
            return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
        }
    };
    /* inflectional-only stemming */
    b = new Analyzer() {

        @Override
        public TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
            return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer, false));
        }
    };
}

Example 57 with Tokenizer

use of org.apache.lucene.analysis.Tokenizer in project lucene-solr by apache.

the class TestIndonesianStemmer method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 58 with Tokenizer

use of org.apache.lucene.analysis.Tokenizer in project lucene-solr by apache.

the class TestIndicNormalizer method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new IndicNormalizationFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 59 with Tokenizer

use of org.apache.lucene.analysis.Tokenizer in project lucene-solr by apache.

the class TestSimplePatternTokenizer method testEmptyStringPatternNoMatch.

public void testEmptyStringPatternNoMatch() throws Exception {
    Tokenizer t = new SimplePatternTokenizer("a*");
    t.setReader(new StringReader("bbb"));
    t.reset();
    assertFalse(t.incrementToken());
}

Example 60 with Tokenizer

use of org.apache.lucene.analysis.Tokenizer in project lucene-solr by apache.

the class TestSimplePatternTokenizer method testBigLookahead.

public void testBigLookahead() throws Exception {
    StringBuilder b = new StringBuilder();
    for (int i = 0; i < 100; i++) {
        b.append('a');
    }
    b.append('b');
    Tokenizer t = new SimplePatternTokenizer(b.toString());
    b = new StringBuilder();
    for (int i = 0; i < 200; i++) {
        b.append('a');
    }
    t.setReader(new StringReader(b.toString()));
    t.reset();
    assertFalse(t.incrementToken());
}