use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.
the class TestSynonymGraphFilter method getAnalyzer.
private Analyzer getAnalyzer(SynonymMap.Builder b, final boolean ignoreCase) throws IOException {
final SynonymMap map = b.build();
return new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
// Make a local variable so testRandomHuge doesn't share it across threads!
SynonymGraphFilter synFilter = new SynonymGraphFilter(tokenizer, map, ignoreCase);
TestSynonymGraphFilter.this.flattenFilter = null;
TestSynonymGraphFilter.this.synFilter = synFilter;
return new TokenStreamComponents(tokenizer, synFilter);
}
};
}
use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.
the class TestSynonymGraphFilter method testZeroSyns.
public void testZeroSyns() throws Exception {
Tokenizer tokenizer = new MockTokenizer();
tokenizer.setReader(new StringReader("aa bb"));
try {
new SynonymGraphFilter(tokenizer, new SynonymMap.Builder(true).build(), true);
fail("did not hit expected exception");
} catch (IllegalArgumentException iae) {
// expected
assertEquals("fst must be non-null", iae.getMessage());
}
}
use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.
the class TestApostropheFilterFactory method testApostrophes.
/**
* Ensure the filter actually removes characters after an apostrophe.
*/
public void testApostrophes() throws Exception {
Reader reader = new StringReader("Türkiye'de 2003'te Van Gölü'nü gördüm");
TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
((Tokenizer) stream).setReader(reader);
stream = tokenFilterFactory("Apostrophe").create(stream);
assertTokenStreamContents(stream, new String[] { "Türkiye", "2003", "Van", "Gölü", "gördüm" });
}
use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.
the class TestSynonymMapFilter method testRecursion.
public void testRecursion() throws Exception {
b = new SynonymMap.Builder(true);
final boolean keepOrig = false;
add("zoo", "zoo", keepOrig);
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
assertAnalyzesTo(a, "zoo zoo $ zoo", new String[] { "zoo", "zoo", "$", "zoo" }, new int[] { 1, 1, 1, 1 });
a.close();
}
use of org.apache.lucene.analysis.MockTokenizer in project lucene-solr by apache.
the class TestSynonymMapFilter method testDontKeepOrig.
public void testDontKeepOrig() throws Exception {
b = new SynonymMap.Builder(true);
add("a b", "foo", false);
final SynonymMap map = b.build();
final Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
assertAnalyzesTo(analyzer, "a b c", new String[] { "foo", "c" }, new int[] { 0, 4 }, new int[] { 3, 5 }, null, new int[] { 1, 1 }, new int[] { 1, 1 }, true);
checkAnalysisConsistency(random(), analyzer, false, "a b c");
analyzer.close();
}
Aggregations