Examples with FieldType org.apache.lucene.document.FieldType used on opensource projects

Example 31 with FieldType

use of org.apache.lucene.document.FieldType in project lucene-solr by apache.

the class FastVectorHighlighterTest method testBooleanPhraseWithSynonym.

public void testBooleanPhraseWithSynonym() throws IOException {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    FieldType type = new FieldType(TextField.TYPE_NOT_STORED);
    type.setStoreTermVectorOffsets(true);
    type.setStoreTermVectorPositions(true);
    type.setStoreTermVectors(true);
    type.freeze();
    Token syn = new Token("httpwwwfacebookcom", 6, 29);
    syn.setPositionIncrement(0);
    CannedTokenStream ts = new CannedTokenStream(new Token("test", 0, 4), new Token("http", 6, 10), syn, new Token("www", 13, 16), new Token("facebook", 17, 25), new Token("com", 26, 29));
    Field field = new Field("field", ts, type);
    doc.add(field);
    doc.add(new StoredField("field", "Test: http://www.facebook.com"));
    writer.addDocument(doc);
    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    IndexReader reader = DirectoryReader.open(writer);
    int docId = 0;
    // query1: match
    PhraseQuery pq = new PhraseQuery("field", "test", "http", "www", "facebook", "com");
    FieldQuery fieldQuery = highlighter.getFieldQuery(pq, reader);
    String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);
    assertEquals("<b>Test: http://www.facebook.com</b>", bestFragments[0]);
    // query2: match
    PhraseQuery pq2 = new PhraseQuery("field", "test", "httpwwwfacebookcom", "www", "facebook", "com");
    fieldQuery = highlighter.getFieldQuery(pq2, reader);
    bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);
    assertEquals("<b>Test: http://www.facebook.com</b>", bestFragments[0]);
    // query3: OR query1 and query2 together
    BooleanQuery.Builder bq = new BooleanQuery.Builder();
    bq.add(pq, BooleanClause.Occur.SHOULD);
    bq.add(pq2, BooleanClause.Occur.SHOULD);
    fieldQuery = highlighter.getFieldQuery(bq.build(), reader);
    bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);
    assertEquals("<b>Test: http://www.facebook.com</b>", bestFragments[0]);
    reader.close();
    writer.close();
    dir.close();
}

Example 32 with FieldType

use of org.apache.lucene.document.FieldType in project lucene-solr by apache.

the class FastVectorHighlighterTest method testPhraseHighlightLongTextTest.

public void testPhraseHighlightLongTextTest() throws IOException {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    FieldType type = new FieldType(TextField.TYPE_STORED);
    type.setStoreTermVectorOffsets(true);
    type.setStoreTermVectorPositions(true);
    type.setStoreTermVectors(true);
    type.freeze();
    Field text = new Field("text", "Netscape was the general name for a series of web browsers originally produced by Netscape Communications Corporation, now a subsidiary of AOL The original browser was once the dominant browser in terms of usage share, but as a result of the first browser war it lost virtually all of its share to Internet Explorer Netscape was discontinued and support for all Netscape browsers and client products was terminated on March 1, 2008 Netscape Navigator was the name of Netscape's web browser from versions 1.0 through 4.8 The first beta release versions of the browser were released in 1994 and known as Mosaic and then Mosaic Netscape until a legal challenge from the National Center for Supercomputing Applications (makers of NCSA Mosaic, which many of Netscape's founders used to develop), led to the name change to Netscape Navigator The company's name also changed from Mosaic Communications Corporation to Netscape Communications Corporation The browser was easily the most advanced...", type);
    doc.add(text);
    writer.addDocument(doc);
    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    IndexReader reader = DirectoryReader.open(writer);
    int docId = 0;
    String field = "text";
    {
        BooleanQuery.Builder query = new BooleanQuery.Builder();
        query.add(new TermQuery(new Term(field, "internet")), Occur.MUST);
        query.add(new TermQuery(new Term(field, "explorer")), Occur.MUST);
        FieldQuery fieldQuery = highlighter.getFieldQuery(query.build(), reader);
        String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, field, 128, 1);
        // highlighted results are centered
        assertEquals(1, bestFragments.length);
        assertEquals("first browser war it lost virtually all of its share to <b>Internet</b> <b>Explorer</b> Netscape was discontinued and support for all Netscape browsers", bestFragments[0]);
    }
    {
        PhraseQuery query = new PhraseQuery(field, "internet", "explorer");
        FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);
        String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, field, 128, 1);
        // highlighted results are centered
        assertEquals(1, bestFragments.length);
        assertEquals("first browser war it lost virtually all of its share to <b>Internet Explorer</b> Netscape was discontinued and support for all Netscape browsers", bestFragments[0]);
    }
    reader.close();
    writer.close();
    dir.close();
}

Example 33 with FieldType

use of org.apache.lucene.document.FieldType in project lucene-solr by apache.

the class FastVectorHighlighterTest method testCustomScoreQueryHighlight.

public void testCustomScoreQueryHighlight() throws IOException {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    FieldType type = new FieldType(TextField.TYPE_STORED);
    type.setStoreTermVectorOffsets(true);
    type.setStoreTermVectorPositions(true);
    type.setStoreTermVectors(true);
    type.freeze();
    Field field = new Field("field", "This is a test where foo is highlighed and should be highlighted", type);
    doc.add(field);
    writer.addDocument(doc);
    FastVectorHighlighter highlighter = new FastVectorHighlighter();
    IndexReader reader = DirectoryReader.open(writer);
    int docId = 0;
    FieldQuery fieldQuery = highlighter.getFieldQuery(new CustomScoreQuery(new TermQuery(new Term("field", "foo"))), reader);
    String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 54, 1);
    // highlighted results are centered
    assertEquals("This is a test where <b>foo</b> is highlighed and should be highlighted", bestFragments[0]);
    bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 52, 1);
    assertEquals("This is a test where <b>foo</b> is highlighed and should be", bestFragments[0]);
    bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 30, 1);
    assertEquals("a test where <b>foo</b> is highlighed", bestFragments[0]);
    reader.close();
    writer.close();
    dir.close();
}

Example 34 with FieldType

use of org.apache.lucene.document.FieldType in project lucene-solr by apache.

the class TestUnifiedHighlighterTermVec method testUserFailedToIndexOffsets.

@Test(expected = IllegalArgumentException.class)
public void testUserFailedToIndexOffsets() throws IOException {
    // note: it's indexed too
    FieldType fieldType = new FieldType(UHTestHelper.tvType);
    fieldType.setStoreTermVectorPositions(random().nextBoolean());
    fieldType.setStoreTermVectorOffsets(false);
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);
    Document doc = new Document();
    doc.add(new Field("body", "term vectors", fieldType));
    iw.addDocument(doc);
    IndexReader ir = iw.getReader();
    iw.close();
    IndexSearcher searcher = newSearcher(ir);
    UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, indexAnalyzer);
    TermQuery query = new TermQuery(new Term("body", "vectors"));
    TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
    try {
        //should throw
        highlighter.highlight("body", query, topDocs, 1);
    } finally {
        ir.close();
    }
}

Example 35 with FieldType

use of org.apache.lucene.document.FieldType in project lucene-solr by apache.

the class TestUnifiedHighlighter method indexSomeFields.

private IndexReader indexSomeFields() throws IOException {
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);
    FieldType ft = new FieldType();
    ft.setIndexOptions(IndexOptions.NONE);
    ft.setTokenized(false);
    ft.setStored(true);
    ft.freeze();
    Field title = new Field("title", "", fieldType);
    Field text = new Field("text", "", fieldType);
    Field category = new Field("category", "", fieldType);
    Document doc = new Document();
    doc.add(title);
    doc.add(text);
    doc.add(category);
    title.setStringValue("This is the title field.");
    text.setStringValue("This is the text field. You can put some text if you want.");
    category.setStringValue("This is the category field.");
    iw.addDocument(doc);
    IndexReader ir = iw.getReader();
    iw.close();
    return ir;
}