Examples with TokenStream org.apache.lucene.analysis.TokenStream used on opensource projects

Example 96 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestRussianLightStemFilterFactory method testStemming.

public void testStemming() throws Exception {
    Reader reader = new StringReader("журналы");
    TokenStream stream = whitespaceMockTokenizer(reader);
    stream = tokenFilterFactory("RussianLightStem").create(stream);
    assertTokenStreamContents(stream, new String[] { "журнал" });
}

Example 97 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class ShingleAnalyzerWrapperTest method testShingleAnalyzerWrapperPhraseQuery.

/*
   * This shows how to construct a phrase query containing shingles.
   */
public void testShingleAnalyzerWrapperPhraseQuery() throws Exception {
    PhraseQuery.Builder builder = new PhraseQuery.Builder();
    try (TokenStream ts = analyzer.tokenStream("content", "this sentence")) {
        int j = -1;
        PositionIncrementAttribute posIncrAtt = ts.addAttribute(PositionIncrementAttribute.class);
        CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
        ts.reset();
        while (ts.incrementToken()) {
            j += posIncrAtt.getPositionIncrement();
            String termText = termAtt.toString();
            builder.add(new Term("content", termText), j);
        }
        ts.end();
    }
    PhraseQuery q = builder.build();
    ScoreDoc[] hits = searcher.search(q, 1000).scoreDocs;
    int[] ranks = new int[] { 0 };
    compareRanks(hits, ranks);
}

Example 98 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestDaitchMokotoffSoundexFilterFactory method testSettingInject.

public void testSettingInject() throws Exception {
    Map<String, String> parameters = new HashMap<>();
    parameters.put("inject", "false");
    DaitchMokotoffSoundexFilterFactory factory = new DaitchMokotoffSoundexFilterFactory(parameters);
    Tokenizer inputStream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
    inputStream.setReader(new StringReader("international"));
    TokenStream filteredStream = factory.create(inputStream);
    assertEquals(DaitchMokotoffSoundexFilter.class, filteredStream.getClass());
    assertTokenStreamContents(filteredStream, new String[] { "063963" });
}

Example 99 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class DoubleMetaphoneFilterTest method testAlternateInjectFalse.

public void testAlternateInjectFalse() throws Exception {
    TokenStream stream = whitespaceMockTokenizer("Kuczewski");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);
    assertTokenStreamContents(filter, new String[] { "KSSK", "KXFS" });
}

Example 100 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class DoubleMetaphoneFilterTest method testNonConvertableStringsWithInject.

public void testNonConvertableStringsWithInject() throws Exception {
    TokenStream stream = whitespaceMockTokenizer("12345 #$%@#^%&");
    TokenStream filter = new DoubleMetaphoneFilter(stream, 8, true);
    assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&" });
}