Examples with TokenStream org.apache.lucene.analysis.TokenStream used on opensource projects

Example 51 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class FuzzySuggesterTest method testInputPathRequired.

public void testInputPathRequired() throws Exception {
    //  SynonymMap.Builder b = new SynonymMap.Builder(false);
    //  b.add(new CharsRef("ab"), new CharsRef("ba"), true);
    //  final SynonymMap map = b.build();
    //  The Analyzer below mimics the functionality of the SynonymAnalyzer
    //  using the above map, so that the suggest module does not need a dependency on the 
    //  synonym module 
    final Analyzer analyzer = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
            return new TokenStreamComponents(tokenizer) {

                int tokenStreamCounter = 0;

                final TokenStream[] tokenStreams = new TokenStream[] { new CannedTokenStream(new Token[] { token("ab", 1, 1), token("ba", 0, 1), token("xc", 1, 1) }), new CannedTokenStream(new Token[] { token("ba", 1, 1), token("xd", 1, 1) }), new CannedTokenStream(new Token[] { token("ab", 1, 1), token("ba", 0, 1), token("x", 1, 1) }) };

                @Override
                public TokenStream getTokenStream() {
                    TokenStream result = tokenStreams[tokenStreamCounter];
                    tokenStreamCounter++;
                    return result;
                }

                @Override
                protected void setReader(final Reader reader) {
                }
            };
        }
    };
    Input[] keys = new Input[] { new Input("ab xc", 50), new Input("ba xd", 50) };
    Directory tempDir = getDirectory();
    FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
    suggester.build(new InputArrayIterator(keys));
    List<LookupResult> results = suggester.lookup("ab x", false, 1);
    assertTrue(results.size() == 1);
    IOUtils.close(analyzer, tempDir);
}

Example 52 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestSuggestStopFilter method testMultipleStopWordsEnd2.

public void testMultipleStopWordsEnd2() throws Exception {
    CharArraySet stopWords = StopFilter.makeStopSet("to", "the", "a");
    Tokenizer stream = new MockTokenizer();
    stream.setReader(new StringReader("go to a the "));
    TokenStream filter = new SuggestStopFilter(stream, stopWords);
    filter = new SuggestStopFilter(stream, stopWords);
    assertTokenStreamContents(filter, new String[] { "go" }, new int[] { 0 }, new int[] { 2 }, null, new int[] { 1 }, null, 12, new boolean[] { false }, true);
}

Example 53 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestSuggestStopFilter method testMultipleStopWords.

public void testMultipleStopWords() throws Exception {
    CharArraySet stopWords = StopFilter.makeStopSet("to", "the", "a");
    Tokenizer stream = new MockTokenizer();
    stream.setReader(new StringReader("go to a the school"));
    TokenStream filter = new SuggestStopFilter(stream, stopWords);
    filter = new SuggestStopFilter(stream, stopWords);
    assertTokenStreamContents(filter, new String[] { "go", "school" }, new int[] { 0, 12 }, new int[] { 2, 18 }, null, new int[] { 1, 4 }, null, 18, new boolean[] { false, false }, true);
}

Example 54 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestSuggestStopFilter method testEndNotStopWord.

public void testEndNotStopWord() throws Exception {
    CharArraySet stopWords = StopFilter.makeStopSet("to");
    Tokenizer stream = new MockTokenizer();
    stream.setReader(new StringReader("go to"));
    TokenStream filter = new SuggestStopFilter(stream, stopWords);
    assertTokenStreamContents(filter, new String[] { "go", "to" }, new int[] { 0, 3 }, new int[] { 2, 5 }, null, new int[] { 1, 1 }, null, 5, new boolean[] { false, true }, true);
}

Example 55 with TokenStream

use of org.apache.lucene.analysis.TokenStream in project lucene-solr by apache.

the class TestSuggestStopFilter method testMidStopWord.

public void testMidStopWord() throws Exception {
    CharArraySet stopWords = StopFilter.makeStopSet("to");
    Tokenizer stream = new MockTokenizer();
    stream.setReader(new StringReader("go to school"));
    TokenStream filter = new SuggestStopFilter(stream, stopWords);
    filter = new SuggestStopFilter(stream, stopWords);
    assertTokenStreamContents(filter, new String[] { "go", "school" }, new int[] { 0, 6 }, new int[] { 2, 12 }, null, new int[] { 1, 2 }, null, 12, new boolean[] { false, false }, true);
}