Examples with StringField org.apache.lucene.document.StringField used on opensource projects

Example 26 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class IndexSearcherWrapperTests method testNoWrap.

public void testNoWrap() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new StringField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    doc.add(new TextField("field", "doc", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    DirectoryReader open = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "_na_", 1));
    IndexSearcher searcher = new IndexSearcher(open);
    assertEquals(1, searcher.search(new TermQuery(new Term("field", "doc")), 1).totalHits);
    searcher.setSimilarity(iwc.getSimilarity());
    IndexSearcherWrapper wrapper = new IndexSearcherWrapper();
    try (Engine.Searcher engineSearcher = new Engine.Searcher("foo", searcher)) {
        final Engine.Searcher wrap = wrapper.wrap(engineSearcher);
        assertSame(wrap, engineSearcher);
    }
    IOUtils.close(open, writer, dir);
}

Example 27 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class NestedSortingTests method testDuel.

public void testDuel() throws Exception {
    final int numDocs = scaledRandomIntBetween(100, 1000);
    for (int i = 0; i < numDocs; ++i) {
        final int numChildren = randomInt(2);
        List<Document> docs = new ArrayList<>(numChildren + 1);
        for (int j = 0; j < numChildren; ++j) {
            Document doc = new Document();
            doc.add(new StringField("f", TestUtil.randomSimpleString(random(), 2), Field.Store.NO));
            doc.add(new StringField("__type", "child", Field.Store.NO));
            docs.add(doc);
        }
        if (randomBoolean()) {
            docs.add(new Document());
        }
        Document parent = new Document();
        parent.add(new StringField("__type", "parent", Field.Store.NO));
        docs.add(parent);
        writer.addDocuments(docs);
        if (rarely()) {
            // we need to have a bit more segments than what RandomIndexWriter would do by default
            DirectoryReader.open(writer).close();
        }
    }
    writer.commit();
    MultiValueMode sortMode = randomFrom(Arrays.asList(MultiValueMode.MIN, MultiValueMode.MAX));
    DirectoryReader reader = DirectoryReader.open(writer);
    reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0));
    IndexSearcher searcher = new IndexSearcher(reader);
    PagedBytesIndexFieldData indexFieldData1 = getForField("f");
    IndexFieldData<?> indexFieldData2 = NoOrdinalsStringFieldDataTests.hideOrdinals(indexFieldData1);
    final String missingValue = randomBoolean() ? null : TestUtil.randomSimpleString(random(), 2);
    final int n = randomIntBetween(1, numDocs + 2);
    final boolean reverse = randomBoolean();
    final TopDocs topDocs1 = getTopDocs(searcher, indexFieldData1, missingValue, sortMode, n, reverse);
    final TopDocs topDocs2 = getTopDocs(searcher, indexFieldData2, missingValue, sortMode, n, reverse);
    for (int i = 0; i < topDocs1.scoreDocs.length; ++i) {
        final FieldDoc fieldDoc1 = (FieldDoc) topDocs1.scoreDocs[i];
        final FieldDoc fieldDoc2 = (FieldDoc) topDocs2.scoreDocs[i];
        assertEquals(fieldDoc1.doc, fieldDoc2.doc);
        assertArrayEquals(fieldDoc1.fields, fieldDoc2.fields);
    }
    searcher.getIndexReader().close();
}

Example 28 with StringField

use of org.apache.lucene.document.StringField in project elasticsearch by elastic.

the class RecoverySourceHandlerTests method testHandleExceptinoOnSendSendFiles.

public void testHandleExceptinoOnSendSendFiles() throws Throwable {
    Settings settings = Settings.builder().put("indices.recovery.concurrent_streams", 1).put("indices.recovery.concurrent_small_file_streams", 1).build();
    final RecoverySettings recoverySettings = new RecoverySettings(settings, service);
    final StartRecoveryRequest request = new StartRecoveryRequest(shardId, new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), new DiscoveryNode("b", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), null, randomBoolean(), randomNonNegativeLong(), randomBoolean() ? SequenceNumbersService.UNASSIGNED_SEQ_NO : 0L);
    Path tempDir = createTempDir();
    Store store = newStore(tempDir, false);
    AtomicBoolean failedEngine = new AtomicBoolean(false);
    RecoverySourceHandler handler = new RecoverySourceHandler(null, null, request, () -> 0L, e -> () -> {
    }, recoverySettings.getChunkSize().bytesAsInt(), Settings.EMPTY) {

        @Override
        protected void failEngine(IOException cause) {
            assertFalse(failedEngine.get());
            failedEngine.set(true);
        }
    };
    Directory dir = store.directory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, newIndexWriterConfig());
    int numDocs = randomIntBetween(10, 100);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(new StringField("id", Integer.toString(i), Field.Store.YES));
        document.add(newField("field", randomUnicodeOfCodepointLengthBetween(1, 10), TextField.TYPE_STORED));
        writer.addDocument(document);
    }
    writer.commit();
    writer.close();
    Store.MetadataSnapshot metadata = store.getMetadata(null);
    List<StoreFileMetaData> metas = new ArrayList<>();
    for (StoreFileMetaData md : metadata) {
        metas.add(md);
    }
    final boolean throwCorruptedIndexException = randomBoolean();
    Store targetStore = newStore(createTempDir(), false);
    try {
        handler.sendFiles(store, metas.toArray(new StoreFileMetaData[0]), (md) -> {
            if (throwCorruptedIndexException) {
                throw new RuntimeException(new CorruptIndexException("foo", "bar"));
            } else {
                throw new RuntimeException("boom");
            }
        });
        fail("exception index");
    } catch (RuntimeException ex) {
        assertNull(ExceptionsHelper.unwrapCorruption(ex));
        if (throwCorruptedIndexException) {
            assertEquals(ex.getMessage(), "[File corruption occurred on recovery but checksums are ok]");
        } else {
            assertEquals(ex.getMessage(), "boom");
        }
    } catch (CorruptIndexException ex) {
        fail("not expected here");
    }
    assertFalse(failedEngine.get());
    IOUtils.close(store, targetStore);
}

Example 29 with StringField

use of org.apache.lucene.document.StringField in project OpenGrok by OpenGrok.

the class AnalyzerGuru method populateDocument.

/**
     * Populate a Lucene document with the required fields.
     *
     * @param doc The document to populate
     * @param file The file to index
     * @param path Where the file is located (from source root)
     * @param fa The analyzer to use on the file
     * @param xrefOut Where to write the xref (possibly {@code null})
     * @throws IOException If an exception occurs while collecting the data
     */
public void populateDocument(Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException {
    String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
    doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
    doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
    try {
        HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
        if (hr != null) {
            doc.add(new TextField(QueryBuilder.HIST, hr));
        // date = hr.getLastCommentDate() //RFE
        }
    } catch (HistoryException e) {
        LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
    }
    doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
    if (path != null) {
        doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
        Project project = Project.getProject(path);
        if (project != null) {
            doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
        }
    }
    if (fa != null) {
        Genre g = fa.getGenre();
        if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
            doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
        }
        fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
        String type = fa.getFileTypeName();
        doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
    }
}

Example 30 with StringField

use of org.apache.lucene.document.StringField in project languagetool by languagetool-org.

the class SentenceSourceIndexer method writeMetaDocuments.

private void writeMetaDocuments() throws IOException {
    Document doc = new Document();
    doc.add(new StringField(MAX_DOC_COUNT_FIELD, MAX_DOC_COUNT_FIELD_VAL, Field.Store.YES));
    doc.add(new StringField(MAX_DOC_COUNT_VALUE, sentenceCount + "", Field.Store.YES));
    indexer.add(doc);
}