Examples with KeywordTokenizer org.apache.lucene.analysis.core.KeywordTokenizer used on opensource projects

Example 21 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestASCIIFoldingFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer, random().nextBoolean()));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 22 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestSynonymMapFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Random random = random();
    final int numIters = atLeast(10);
    for (int i = 0; i < numIters; i++) {
        b = new SynonymMap.Builder(random.nextBoolean());
        final int numEntries = atLeast(10);
        for (int j = 0; j < numEntries; j++) {
            add(randomNonEmptyString(), randomNonEmptyString(), random.nextBoolean());
        }
        final SynonymMap map = b.build();
        final boolean ignoreCase = random.nextBoolean();
        final Analyzer analyzer = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new KeywordTokenizer();
                return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
            }
        };
        checkAnalysisConsistency(random, analyzer, random.nextBoolean(), "");
        analyzer.close();
    }
}

Example 23 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestGermanStemFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new GermanStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 24 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestFrenchLightStemFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new FrenchLightStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 25 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestFrenchMinimalStemFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new FrenchMinimalStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}