Examples with StandardAnalyzer org.apache.lucene.analysis.standard.StandardAnalyzer used on opensource projects

Example 11 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class TermVectorsUnitTests method writeStandardTermVector.

private void writeStandardTermVector(TermVectorsResponse outResponse) throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
    conf.setOpenMode(OpenMode.CREATE);
    IndexWriter writer = new IndexWriter(dir, conf);
    FieldType type = new FieldType(TextField.TYPE_STORED);
    type.setStoreTermVectorOffsets(true);
    type.setStoreTermVectorPayloads(false);
    type.setStoreTermVectorPositions(true);
    type.setStoreTermVectors(true);
    type.freeze();
    Document d = new Document();
    d.add(new Field("id", "abc", StringField.TYPE_STORED));
    d.add(new Field("title", "the1 quick brown fox jumps over  the1 lazy dog", type));
    d.add(new Field("desc", "the1 quick brown fox jumps over  the1 lazy dog", type));
    writer.updateDocument(new Term("id", "abc"), d);
    writer.commit();
    writer.close();
    DirectoryReader dr = DirectoryReader.open(dir);
    IndexSearcher s = new IndexSearcher(dr);
    TopDocs search = s.search(new TermQuery(new Term("id", "abc")), 1);
    ScoreDoc[] scoreDocs = search.scoreDocs;
    int doc = scoreDocs[0].doc;
    Fields termVectors = dr.getTermVectors(doc);
    EnumSet<Flag> flags = EnumSet.of(Flag.Positions, Flag.Offsets);
    outResponse.setFields(termVectors, null, flags, termVectors);
    dr.close();
    dir.close();
}

Example 12 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class CustomUnifiedHighlighterTests method testMultiPhrasePrefixQuery.

public void testMultiPhrasePrefixQuery() throws Exception {
    final String[] inputs = { "The quick brown fox." };
    final String[] outputs = { "The <b>quick</b> <b>brown</b> <b>fox</b>." };
    MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery();
    query.add(new Term("text", "quick"));
    query.add(new Term("text", "brown"));
    query.add(new Term("text", "fo"));
    assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT, BreakIterator.getSentenceInstance(Locale.ROOT), 0, outputs);
}

Example 13 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class CustomUnifiedHighlighterTests method testSentenceBoundedBreakIterator.

public void testSentenceBoundedBreakIterator() throws Exception {
    final String[] inputs = { "The quick brown fox in a long sentence with another quick brown fox. " + "Another sentence with brown fox." };
    final String[] outputs = { "The <b>quick</b> <b>brown</b>", "<b>fox</b> in a long", "with another <b>quick</b>", "<b>brown</b> <b>fox</b>.", "sentence with <b>brown</b>", "<b>fox</b>." };
    BooleanQuery query = new BooleanQuery.Builder().add(new TermQuery(new Term("text", "quick")), BooleanClause.Occur.SHOULD).add(new TermQuery(new Term("text", "brown")), BooleanClause.Occur.SHOULD).add(new TermQuery(new Term("text", "fox")), BooleanClause.Occur.SHOULD).build();
    assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT, BoundedBreakIteratorScanner.getSentence(Locale.ROOT, 10), 0, outputs);
}

Example 14 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class CustomUnifiedHighlighterTests method testSimple.

public void testSimple() throws Exception {
    final String[] inputs = { "This is a test. Just a test1 highlighting from unified highlighter.", "This is the second highlighting value to perform highlighting on a longer text that gets scored lower.", "This is highlighting the third short highlighting value.", "Just a test4 highlighting from unified highlighter." };
    String[] expectedPassages = { "Just a test1 <b>highlighting</b> from unified highlighter.", "This is the second <b>highlighting</b> value to perform <b>highlighting</b> on a" + " longer text that gets scored lower.", "This is <b>highlighting</b> the third short <b>highlighting</b> value.", "Just a test4 <b>highlighting</b> from unified highlighter." };
    Query query = new TermQuery(new Term("text", "highlighting"));
    assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT, BreakIterator.getSentenceInstance(Locale.ROOT), 0, expectedPassages);
}

Example 15 with StandardAnalyzer

use of org.apache.lucene.analysis.standard.StandardAnalyzer in project elasticsearch by elastic.

the class CustomUnifiedHighlighterTests method testRepeat.

public void testRepeat() throws Exception {
    final String[] inputs = { "Fun  fun fun  fun  fun  fun  fun  fun  fun  fun" };
    final String[] outputs = { "<b>Fun</b>  <b>fun</b> <b>fun</b>", "<b>fun</b>  <b>fun</b>", "<b>fun</b>  <b>fun</b>  <b>fun</b>", "<b>fun</b>  <b>fun</b>" };
    Query query = new TermQuery(new Term("text", "fun"));
    assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT, BoundedBreakIteratorScanner.getSentence(Locale.ROOT, 10), 0, outputs);
    query = new PhraseQuery.Builder().add(new Term("text", "fun")).add(new Term("text", "fun")).build();
    assertHighlightOneDoc("text", inputs, new StandardAnalyzer(), query, Locale.ROOT, BoundedBreakIteratorScanner.getSentence(Locale.ROOT, 10), 0, outputs);
}