Examples with KeywordTokenizer org.apache.lucene.analysis.core.KeywordTokenizer used on opensource projects

Example 1 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project elasticsearch by elastic.

the class SimpleIcuCollationTokenFilterTests method assertCollation.

private void assertCollation(TokenFilterFactory factory, String string1, String string2, int comparison) throws IOException {
    Tokenizer tokenizer = new KeywordTokenizer();
    tokenizer.setReader(new StringReader(string1));
    TokenStream stream1 = factory.create(tokenizer);
    tokenizer = new KeywordTokenizer();
    tokenizer.setReader(new StringReader(string2));
    TokenStream stream2 = factory.create(tokenizer);
    assertCollation(stream1, stream2, comparison);
}

Example 2 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestIndonesianStemmer method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 3 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestIndicNormalizer method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new IndicNormalizationFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 4 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestPortugueseMinimalStemFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new PortugueseMinimalStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 5 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestPortugueseLightStemFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new PortugueseLightStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}