Examples with BinaryToken org.apache.lucene.analysis.CannedBinaryTokenStream.BinaryToken used on opensource projects

Example 1 with BinaryToken

use of org.apache.lucene.analysis.CannedBinaryTokenStream.BinaryToken in project lucene-solr by apache.

the class AnalyzingSuggesterTest method test0ByteKeys.

public void test0ByteKeys() throws Exception {
    final Analyzer a = new Analyzer() {

        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
            Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
            return new TokenStreamComponents(tokenizer) {

                int tokenStreamCounter = 0;

                final TokenStream[] tokenStreams = new TokenStream[] { new CannedBinaryTokenStream(new BinaryToken[] { token(new BytesRef(new byte[] { 0x0, 0x0, 0x0 })) }), new CannedBinaryTokenStream(new BinaryToken[] { token(new BytesRef(new byte[] { 0x0, 0x0 })) }), new CannedBinaryTokenStream(new BinaryToken[] { token(new BytesRef(new byte[] { 0x0, 0x0, 0x0 })) }), new CannedBinaryTokenStream(new BinaryToken[] { token(new BytesRef(new byte[] { 0x0, 0x0 })) }) };

                @Override
                public TokenStream getTokenStream() {
                    TokenStream result = tokenStreams[tokenStreamCounter];
                    tokenStreamCounter++;
                    return result;
                }

                @Override
                protected void setReader(final Reader reader) {
                }
            };
        }
    };
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
    suggester.build(new InputArrayIterator(new Input[] { new Input("a a", 50), new Input("a b", 50) }));
    IOUtils.close(a, tempDir);
}