Examples with KeywordTokenizer org.apache.lucene.analysis.core.KeywordTokenizer used on opensource projects

Example 16 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestItalianLightStemFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new ItalianLightStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 17 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestHunspellStemFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, dictionary));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 18 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestHunspellStemFilter method testIgnoreCaseNoSideEffects.

public void testIgnoreCaseNoSideEffects() throws Exception {
    final Dictionary d;
    // no multiple try-with to workaround bogus VerifyError
    InputStream affixStream = TestStemmer.class.getResourceAsStream("simple.aff");
    InputStream dictStream = TestStemmer.class.getResourceAsStream("simple.dic");
    Directory tempDir = getDirectory();
    try {
        d = new Dictionary(tempDir, "dictionary", affixStream, Collections.singletonList(dictStream), true);
    } finally {
        IOUtils.closeWhileHandlingException(affixStream, dictStream);
    }
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, d));
        }
    };
    checkOneTerm(a, "NoChAnGy", "NoChAnGy");
    a.close();
    tempDir.close();
}

Example 19 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestArabicNormalizationFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new ArabicNormalizationFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}

Example 20 with KeywordTokenizer

use of org.apache.lucene.analysis.core.KeywordTokenizer in project lucene-solr by apache.

the class TestBrazilianAnalyzer method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new KeywordTokenizer();
            return new TokenStreamComponents(tokenizer, new BrazilianStemFilter(tokenizer));
        }
    };
    checkOneTerm(a, "", "");
    a.close();
}