Examples with Token org.apache.lucene.analysis.Token used on opensource projects

Example 11 with Token

use of org.apache.lucene.analysis.Token in project lucene-solr by apache.

the class TestMultiPhraseQuery method testZeroPosIncr.

public void testZeroPosIncr() throws IOException {
    Directory dir = new RAMDirectory();
    final Token[] tokens = new Token[3];
    tokens[0] = new Token();
    tokens[0].append("a");
    tokens[0].setPositionIncrement(1);
    tokens[1] = new Token();
    tokens[1].append("b");
    tokens[1].setPositionIncrement(0);
    tokens[2] = new Token();
    tokens[2].append("c");
    tokens[2].setPositionIncrement(0);
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    doc.add(new TextField("field", new CannedTokenStream(tokens)));
    writer.addDocument(doc);
    doc = new Document();
    doc.add(new TextField("field", new CannedTokenStream(tokens)));
    writer.addDocument(doc);
    IndexReader r = writer.getReader();
    writer.close();
    IndexSearcher s = newSearcher(r);
    MultiPhraseQuery.Builder mpqb = new MultiPhraseQuery.Builder();
    // case):
    if (true) {
        mpqb.add(new Term[] { new Term("field", "b"), new Term("field", "c") }, 0);
        mpqb.add(new Term[] { new Term("field", "a") }, 0);
    } else {
        mpqb.add(new Term[] { new Term("field", "a") }, 0);
        mpqb.add(new Term[] { new Term("field", "b"), new Term("field", "c") }, 0);
    }
    TopDocs hits = s.search(mpqb.build(), 2);
    assertEquals(2, hits.totalHits);
    assertEquals(hits.scoreDocs[0].score, hits.scoreDocs[1].score, 1e-5);
    /*
    for(int hit=0;hit<hits.totalHits;hit++) {
      ScoreDoc sd = hits.scoreDocs[hit];
      System.out.println("  hit doc=" + sd.doc + " score=" + sd.score);
    }
    */
    r.close();
    dir.close();
}

Example 12 with Token

use of org.apache.lucene.analysis.Token in project lucene-solr by apache.

the class TestMultiPhraseQuery method testZeroPosIncrSloppyPqAnd.

/**
   * PQ AND Mode - Manually creating a phrase query
   */
public void testZeroPosIncrSloppyPqAnd() throws IOException {
    PhraseQuery.Builder builder = new PhraseQuery.Builder();
    int pos = -1;
    for (Token tap : INCR_0_QUERY_TOKENS_AND) {
        pos += tap.getPositionIncrement();
        builder.add(new Term("field", tap.toString()), pos);
    }
    builder.setSlop(0);
    doTestZeroPosIncrSloppy(builder.build(), 0);
    builder.setSlop(1);
    doTestZeroPosIncrSloppy(builder.build(), 0);
    builder.setSlop(2);
    doTestZeroPosIncrSloppy(builder.build(), 1);
}

Example 13 with Token

use of org.apache.lucene.analysis.Token in project lucene-solr by apache.

the class TestTrimFilter method testTrim.

public void testTrim() throws Exception {
    char[] a = " a ".toCharArray();
    char[] b = "b   ".toCharArray();
    char[] ccc = "cCc".toCharArray();
    char[] whitespace = "   ".toCharArray();
    char[] empty = "".toCharArray();
    TokenStream ts = new CannedTokenStream(new Token(new String(a, 0, a.length), 1, 5), new Token(new String(b, 0, b.length), 6, 10), new Token(new String(ccc, 0, ccc.length), 11, 15), new Token(new String(whitespace, 0, whitespace.length), 16, 20), new Token(new String(empty, 0, empty.length), 21, 21));
    ts = new TrimFilter(ts);
    assertTokenStreamContents(ts, new String[] { "a", "b", "cCc", "", "" });
}

Example 14 with Token

use of org.apache.lucene.analysis.Token in project lucene-solr by apache.

the class TestRemoveDuplicatesTokenFilter method tok.

public static Token tok(int pos, String t, int start, int end) {
    Token tok = new Token(t, start, end);
    tok.setPositionIncrement(pos);
    return tok;
}

Example 15 with Token

use of org.apache.lucene.analysis.Token in project lucene-solr by apache.

the class TestRemoveDuplicatesTokenFilterFactory method tok.

public static Token tok(int pos, String t, int start, int end) {
    Token tok = new Token(t, start, end);
    tok.setPositionIncrement(pos);
    return tok;
}