Examples with TextField org.apache.lucene.document.TextField used on opensource projects

Example 1 with TextField

use of org.apache.lucene.document.TextField in project zeppelin by apache.

the class LuceneSearch method newDocument.

/**
   * If paragraph is not null, indexes code in the paragraph, otherwise indexes
   * the notebook name.
   *
   * @param id id of the document, different for Note name and paragraph
   * @param noteName name of the note
   * @param p paragraph
   * @return
   */
private Document newDocument(String id, String noteName, Paragraph p) {
    Document doc = new Document();
    Field pathField = new StringField(ID_FIELD, id, Field.Store.YES);
    doc.add(pathField);
    doc.add(new StringField("title", noteName, Field.Store.YES));
    if (null != p) {
        doc.add(new TextField(SEARCH_FIELD_TEXT, p.getText(), Field.Store.YES));
        if (p.getTitle() != null) {
            doc.add(new TextField(SEARCH_FIELD_TITLE, p.getTitle(), Field.Store.YES));
        }
        Date date = p.getDateStarted() != null ? p.getDateStarted() : p.getDateCreated();
        doc.add(new LongField("modified", date.getTime(), Field.Store.NO));
    } else {
        doc.add(new TextField(SEARCH_FIELD_TEXT, noteName, Field.Store.YES));
    }
    return doc;
}

Example 2 with TextField

use of org.apache.lucene.document.TextField in project elasticsearch by elastic.

the class DiversifiedSamplerTests method testDiversifiedSampler.

public void testDiversifiedSampler() throws Exception {
    String[] data = { // "id,cat,name,price,inStock,author_t,series_t,sequence_i,genre_s,genre_id",
    "0553573403,book,A Game of Thrones,7.99,true,George R.R. Martin,A Song of Ice and Fire,1,fantasy,0", "0553579908,book,A Clash of Kings,7.99,true,George R.R. Martin,A Song of Ice and Fire,2,fantasy,0", "055357342X,book,A Storm of Swords,7.99,true,George R.R. Martin,A Song of Ice and Fire,3,fantasy,0", "0553293354,book,Foundation,17.99,true,Isaac Asimov,Foundation Novels,1,scifi,1", "0812521390,book,The Black Company,6.99,false,Glen Cook,The Chronicles of The Black Company,1,fantasy,0", "0812550706,book,Ender's Game,6.99,true,Orson Scott Card,Ender,1,scifi,1", "0441385532,book,Jhereg,7.95,false,Steven Brust,Vlad Taltos,1,fantasy,0", "0380014300,book,Nine Princes In Amber,6.99,true,Roger Zelazny,the Chronicles of Amber,1,fantasy,0", "0805080481,book,The Book of Three,5.99,true,Lloyd Alexander,The Chronicles of Prydain,1,fantasy,0", "080508049X,book,The Black Cauldron,5.99,true,Lloyd Alexander,The Chronicles of Prydain,2,fantasy,0" };
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    for (String entry : data) {
        String[] parts = entry.split(",");
        Document document = new Document();
        document.add(new SortedDocValuesField("id", new BytesRef(parts[0])));
        document.add(new StringField("cat", parts[1], Field.Store.NO));
        document.add(new TextField("name", parts[2], Field.Store.NO));
        document.add(new DoubleDocValuesField("price", Double.valueOf(parts[3])));
        document.add(new StringField("inStock", parts[4], Field.Store.NO));
        document.add(new StringField("author", parts[5], Field.Store.NO));
        document.add(new StringField("series", parts[6], Field.Store.NO));
        document.add(new StringField("sequence", parts[7], Field.Store.NO));
        document.add(new SortedDocValuesField("genre", new BytesRef(parts[8])));
        document.add(new NumericDocValuesField("genre_id", Long.valueOf(parts[9])));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = new IndexSearcher(indexReader);
    MappedFieldType genreFieldType = new KeywordFieldMapper.KeywordFieldType();
    genreFieldType.setName("genre");
    genreFieldType.setHasDocValues(true);
    Consumer<InternalSampler> verify = result -> {
        Terms terms = result.getAggregations().get("terms");
        assertEquals(2, terms.getBuckets().size());
        assertEquals("0805080481", terms.getBuckets().get(0).getKeyAsString());
        assertEquals("0812550706", terms.getBuckets().get(1).getKeyAsString());
    };
    testCase(indexSearcher, genreFieldType, "map", verify);
    testCase(indexSearcher, genreFieldType, "global_ordinals", verify);
    testCase(indexSearcher, genreFieldType, "bytes_hash", verify);
    genreFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    genreFieldType.setName("genre_id");
    testCase(indexSearcher, genreFieldType, null, verify);
    // wrong field:
    genreFieldType = new KeywordFieldMapper.KeywordFieldType();
    genreFieldType.setName("wrong_field");
    genreFieldType.setHasDocValues(true);
    testCase(indexSearcher, genreFieldType, null, result -> {
        Terms terms = result.getAggregations().get("terms");
        assertEquals(1, terms.getBuckets().size());
        assertEquals("0805080481", terms.getBuckets().get(0).getKeyAsString());
    });
    indexReader.close();
    directory.close();
}

Example 3 with TextField

use of org.apache.lucene.document.TextField in project che by eclipse.

the class LuceneSearcher method createDocument.

protected Document createDocument(VirtualFile virtualFile, Reader reader) throws ServerException {
    final Document doc = new Document();
    doc.add(new StringField(PATH_FIELD, virtualFile.getPath().toString(), Field.Store.YES));
    doc.add(new TextField(NAME_FIELD, virtualFile.getName(), Field.Store.YES));
    if (reader != null) {
        doc.add(new TextField(TEXT_FIELD, reader));
    }
    return doc;
}

Example 4 with TextField

use of org.apache.lucene.document.TextField in project elasticsearch by elastic.

the class FreqTermsEnumTests method setUp.

@Before
@Override
public void setUp() throws Exception {
    super.setUp();
    referenceAll = new HashMap<>();
    referenceNotDeleted = new HashMap<>();
    referenceFilter = new HashMap<>();
    Directory dir = newDirectory();
    // use keyword analyzer we rely on the stored field holding the exact term.
    IndexWriterConfig conf = newIndexWriterConfig(new KeywordAnalyzer());
    if (frequently()) {
        // we don't want to do any merges, so we won't expunge deletes
        conf.setMergePolicy(NoMergePolicy.INSTANCE);
    }
    iw = new IndexWriter(dir, conf);
    terms = new String[scaledRandomIntBetween(10, 300)];
    for (int i = 0; i < terms.length; i++) {
        terms[i] = randomAsciiOfLength(5);
    }
    int numberOfDocs = scaledRandomIntBetween(30, 300);
    Document[] docs = new Document[numberOfDocs];
    for (int i = 0; i < numberOfDocs; i++) {
        Document doc = new Document();
        doc.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        docs[i] = doc;
        for (String term : terms) {
            if (randomBoolean()) {
                continue;
            }
            int freq = randomIntBetween(1, 3);
            for (int j = 0; j < freq; j++) {
                doc.add(new TextField("field", term, Field.Store.YES));
            }
        }
    }
    for (int i = 0; i < docs.length; i++) {
        Document doc = docs[i];
        iw.addDocument(doc);
        if (rarely()) {
            iw.commit();
        }
    }
    Set<String> deletedIds = new HashSet<>();
    for (int i = 0; i < docs.length; i++) {
        Document doc = docs[i];
        if (randomInt(5) == 2) {
            Term idTerm = new Term("id", doc.getField("id").stringValue());
            deletedIds.add(idTerm.text());
            iw.deleteDocuments(idTerm);
        }
    }
    for (String term : terms) {
        referenceAll.put(term, new FreqHolder());
        referenceFilter.put(term, new FreqHolder());
        referenceNotDeleted.put(term, new FreqHolder());
    }
    // now go over each doc, build the relevant references and filter
    reader = DirectoryReader.open(iw);
    List<BytesRef> filterTerms = new ArrayList<>();
    for (int docId = 0; docId < reader.maxDoc(); docId++) {
        Document doc = reader.document(docId);
        addFreqs(doc, referenceAll);
        if (!deletedIds.contains(doc.getField("id").stringValue())) {
            addFreqs(doc, referenceNotDeleted);
            if (randomBoolean()) {
                filterTerms.add(new BytesRef(doc.getField("id").stringValue()));
                addFreqs(doc, referenceFilter);
            }
        }
    }
    filter = new TermInSetQuery("id", filterTerms);
}

Example 5 with TextField

use of org.apache.lucene.document.TextField in project elasticsearch by elastic.

the class MoreLikeThisQueryTests method testSimple.

public void testSimple() throws Exception {
    Directory dir = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
    indexWriter.commit();
    Document document = new Document();
    document.add(new TextField("_id", "1", Field.Store.YES));
    document.add(new TextField("text", "lucene", Field.Store.YES));
    indexWriter.addDocument(document);
    document = new Document();
    document.add(new TextField("_id", "2", Field.Store.YES));
    document.add(new TextField("text", "lucene release", Field.Store.YES));
    indexWriter.addDocument(document);
    IndexReader reader = DirectoryReader.open(indexWriter);
    IndexSearcher searcher = new IndexSearcher(reader);
    MoreLikeThisQuery mltQuery = new MoreLikeThisQuery("lucene", new String[] { "text" }, Lucene.STANDARD_ANALYZER);
    mltQuery.setLikeText("lucene");
    mltQuery.setMinTermFrequency(1);
    mltQuery.setMinDocFreq(1);
    long count = searcher.count(mltQuery);
    assertThat(count, equalTo(2L));
    reader.close();
    indexWriter.close();
}