Search in sources :

Example 1 with RefinedSoundex

use of org.apache.commons.codec.language.RefinedSoundex in project lucene-solr by apache.

the class TestPhoneticFilter method testRandomStrings.

/** blast some random strings through the analyzer */
public void testRandomStrings() throws IOException {
    Encoder[] encoders = new Encoder[] { new Metaphone(), new DoubleMetaphone(), new Soundex(), new RefinedSoundex(), new Caverphone2() };
    for (final Encoder e : encoders) {
        Analyzer a = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, false));
            }
        };
        checkRandomData(random(), a, 1000 * RANDOM_MULTIPLIER);
        a.close();
        Analyzer b = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
                return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, false));
            }
        };
        checkRandomData(random(), b, 1000 * RANDOM_MULTIPLIER);
        b.close();
    }
}
Also used : RefinedSoundex(org.apache.commons.codec.language.RefinedSoundex) DoubleMetaphone(org.apache.commons.codec.language.DoubleMetaphone) Metaphone(org.apache.commons.codec.language.Metaphone) Caverphone2(org.apache.commons.codec.language.Caverphone2) Analyzer(org.apache.lucene.analysis.Analyzer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer) RefinedSoundex(org.apache.commons.codec.language.RefinedSoundex) Soundex(org.apache.commons.codec.language.Soundex) DoubleMetaphone(org.apache.commons.codec.language.DoubleMetaphone) Encoder(org.apache.commons.codec.Encoder) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer) KeywordTokenizer(org.apache.lucene.analysis.core.KeywordTokenizer)

Example 2 with RefinedSoundex

use of org.apache.commons.codec.language.RefinedSoundex in project lucene-solr by apache.

the class TestPhoneticFilter method testAlgorithms.

public void testAlgorithms() throws Exception {
    assertAlgorithm(new Metaphone(), true, "aaa bbb ccc easgasg", new String[] { "A", "aaa", "B", "bbb", "KKK", "ccc", "ESKS", "easgasg" });
    assertAlgorithm(new Metaphone(), false, "aaa bbb ccc easgasg", new String[] { "A", "B", "KKK", "ESKS" });
    assertAlgorithm(new DoubleMetaphone(), true, "aaa bbb ccc easgasg", new String[] { "A", "aaa", "PP", "bbb", "KK", "ccc", "ASKS", "easgasg" });
    assertAlgorithm(new DoubleMetaphone(), false, "aaa bbb ccc easgasg", new String[] { "A", "PP", "KK", "ASKS" });
    assertAlgorithm(new Soundex(), true, "aaa bbb ccc easgasg", new String[] { "A000", "aaa", "B000", "bbb", "C000", "ccc", "E220", "easgasg" });
    assertAlgorithm(new Soundex(), false, "aaa bbb ccc easgasg", new String[] { "A000", "B000", "C000", "E220" });
    assertAlgorithm(new RefinedSoundex(), true, "aaa bbb ccc easgasg", new String[] { "A0", "aaa", "B1", "bbb", "C3", "ccc", "E034034", "easgasg" });
    assertAlgorithm(new RefinedSoundex(), false, "aaa bbb ccc easgasg", new String[] { "A0", "B1", "C3", "E034034" });
    assertAlgorithm(new Caverphone2(), true, "Darda Karleen Datha Carlene", new String[] { "TTA1111111", "Darda", "KLN1111111", "Karleen", "TTA1111111", "Datha", "KLN1111111", "Carlene" });
    assertAlgorithm(new Caverphone2(), false, "Darda Karleen Datha Carlene", new String[] { "TTA1111111", "KLN1111111", "TTA1111111", "KLN1111111" });
    assertAlgorithm(new Nysiis(), true, "aaa bbb ccc easgasg", new String[] { "A", "aaa", "B", "bbb", "C", "ccc", "EASGAS", "easgasg" });
    assertAlgorithm(new Nysiis(), false, "aaa bbb ccc easgasg", new String[] { "A", "B", "C", "EASGAS" });
}
Also used : RefinedSoundex(org.apache.commons.codec.language.RefinedSoundex) Soundex(org.apache.commons.codec.language.Soundex) DoubleMetaphone(org.apache.commons.codec.language.DoubleMetaphone) RefinedSoundex(org.apache.commons.codec.language.RefinedSoundex) Nysiis(org.apache.commons.codec.language.Nysiis) DoubleMetaphone(org.apache.commons.codec.language.DoubleMetaphone) Metaphone(org.apache.commons.codec.language.Metaphone) Caverphone2(org.apache.commons.codec.language.Caverphone2)

Example 3 with RefinedSoundex

use of org.apache.commons.codec.language.RefinedSoundex in project lucene-solr by apache.

the class TestPhoneticFilter method testEmptyTerm.

public void testEmptyTerm() throws IOException {
    Encoder[] encoders = new Encoder[] { new Metaphone(), new DoubleMetaphone(), new Soundex(), new RefinedSoundex(), new Caverphone2() };
    for (final Encoder e : encoders) {
        Analyzer a = new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new KeywordTokenizer();
                return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, random().nextBoolean()));
            }
        };
        checkOneTerm(a, "", "");
        a.close();
    }
}
Also used : RefinedSoundex(org.apache.commons.codec.language.RefinedSoundex) DoubleMetaphone(org.apache.commons.codec.language.DoubleMetaphone) Metaphone(org.apache.commons.codec.language.Metaphone) Caverphone2(org.apache.commons.codec.language.Caverphone2) Analyzer(org.apache.lucene.analysis.Analyzer) KeywordTokenizer(org.apache.lucene.analysis.core.KeywordTokenizer) RefinedSoundex(org.apache.commons.codec.language.RefinedSoundex) Soundex(org.apache.commons.codec.language.Soundex) DoubleMetaphone(org.apache.commons.codec.language.DoubleMetaphone) Encoder(org.apache.commons.codec.Encoder) Tokenizer(org.apache.lucene.analysis.Tokenizer) MockTokenizer(org.apache.lucene.analysis.MockTokenizer) KeywordTokenizer(org.apache.lucene.analysis.core.KeywordTokenizer)

Aggregations

Caverphone2 (org.apache.commons.codec.language.Caverphone2)3 DoubleMetaphone (org.apache.commons.codec.language.DoubleMetaphone)3 Metaphone (org.apache.commons.codec.language.Metaphone)3 RefinedSoundex (org.apache.commons.codec.language.RefinedSoundex)3 Soundex (org.apache.commons.codec.language.Soundex)3 Encoder (org.apache.commons.codec.Encoder)2 Analyzer (org.apache.lucene.analysis.Analyzer)2 MockTokenizer (org.apache.lucene.analysis.MockTokenizer)2 Tokenizer (org.apache.lucene.analysis.Tokenizer)2 KeywordTokenizer (org.apache.lucene.analysis.core.KeywordTokenizer)2 Nysiis (org.apache.commons.codec.language.Nysiis)1