Examples with NumericDocValuesField org.apache.lucene.document.NumericDocValuesField used on opensource projects

Example 61 with NumericDocValuesField

use of org.apache.lucene.document.NumericDocValuesField in project lucene-solr by apache.

the class TestMixedDocValuesUpdates method testManyReopensAndFields.

public void testManyReopensAndFields() throws Exception {
    Directory dir = newDirectory();
    final Random random = random();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random));
    LogMergePolicy lmp = newLogMergePolicy();
    // merge often
    lmp.setMergeFactor(3);
    conf.setMergePolicy(lmp);
    IndexWriter writer = new IndexWriter(dir, conf);
    final boolean isNRT = random.nextBoolean();
    DirectoryReader reader;
    if (isNRT) {
        reader = DirectoryReader.open(writer);
    } else {
        writer.commit();
        reader = DirectoryReader.open(dir);
    }
    // 3-7
    final int numFields = random.nextInt(4) + 3;
    // 1-3
    final int numNDVFields = random.nextInt(numFields / 2) + 1;
    final long[] fieldValues = new long[numFields];
    for (int i = 0; i < fieldValues.length; i++) {
        fieldValues[i] = 1;
    }
    int numRounds = atLeast(15);
    int docID = 0;
    for (int i = 0; i < numRounds; i++) {
        int numDocs = atLeast(5);
        // System.out.println("TEST: round=" + i + ", numDocs=" + numDocs);
        for (int j = 0; j < numDocs; j++) {
            Document doc = new Document();
            doc.add(new StringField("id", "doc-" + docID, Store.NO));
            // update key
            doc.add(new StringField("key", "all", Store.NO));
            // add all fields with their current value
            for (int f = 0; f < fieldValues.length; f++) {
                if (f < numNDVFields) {
                    doc.add(new NumericDocValuesField("f" + f, fieldValues[f]));
                } else {
                    doc.add(new BinaryDocValuesField("f" + f, TestBinaryDocValuesUpdates.toBytes(fieldValues[f])));
                }
            }
            writer.addDocument(doc);
            ++docID;
        }
        int fieldIdx = random.nextInt(fieldValues.length);
        String updateField = "f" + fieldIdx;
        if (fieldIdx < numNDVFields) {
            writer.updateNumericDocValue(new Term("key", "all"), updateField, ++fieldValues[fieldIdx]);
        } else {
            writer.updateBinaryDocValue(new Term("key", "all"), updateField, TestBinaryDocValuesUpdates.toBytes(++fieldValues[fieldIdx]));
        }
        if (random.nextDouble() < 0.2) {
            // might also delete an already deleted document, ok!
            int deleteDoc = random.nextInt(docID);
            writer.deleteDocuments(new Term("id", "doc-" + deleteDoc));
        //        System.out.println("[" + Thread.currentThread().getName() + "]: deleted document: doc-" + deleteDoc);
        }
        // verify reader
        if (!isNRT) {
            writer.commit();
        }
        //      System.out.println("[" + Thread.currentThread().getName() + "]: reopen reader: " + reader);
        DirectoryReader newReader = DirectoryReader.openIfChanged(reader);
        assertNotNull(newReader);
        reader.close();
        reader = newReader;
        //      System.out.println("[" + Thread.currentThread().getName() + "]: reopened reader: " + reader);
        // we delete at most one document per round
        assertTrue(reader.numDocs() > 0);
        for (LeafReaderContext context : reader.leaves()) {
            LeafReader r = context.reader();
            //        System.out.println(((SegmentReader) r).getSegmentName());
            Bits liveDocs = r.getLiveDocs();
            for (int field = 0; field < fieldValues.length; field++) {
                String f = "f" + field;
                BinaryDocValues bdv = r.getBinaryDocValues(f);
                NumericDocValues ndv = r.getNumericDocValues(f);
                if (field < numNDVFields) {
                    assertNotNull(ndv);
                    assertNull(bdv);
                } else {
                    assertNull(ndv);
                    assertNotNull(bdv);
                }
                int maxDoc = r.maxDoc();
                for (int doc = 0; doc < maxDoc; doc++) {
                    if (liveDocs == null || liveDocs.get(doc)) {
                        //              System.out.println("doc=" + (doc + context.docBase) + " f='" + f + "' vslue=" + getValue(bdv, doc, scratch));
                        if (field < numNDVFields) {
                            assertEquals(doc, ndv.advance(doc));
                            assertEquals("invalid numeric value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], ndv.longValue());
                        } else {
                            assertEquals(doc, bdv.advance(doc));
                            assertEquals("invalid binary value for doc=" + doc + ", field=" + f + ", reader=" + r, fieldValues[field], TestBinaryDocValuesUpdates.getValue(bdv));
                        }
                    }
                }
            }
        }
    //      System.out.println();
    }
    writer.close();
    IOUtils.close(reader, dir);
}

Example 62 with NumericDocValuesField

use of org.apache.lucene.document.NumericDocValuesField in project lucene-solr by apache.

the class TestDocValuesQueries method doTestDuelPointRangeNumericRangeQuery.

private void doTestDuelPointRangeNumericRangeQuery(boolean sortedNumeric, int maxValuesPerDoc) throws IOException {
    final int iters = atLeast(10);
    for (int iter = 0; iter < iters; ++iter) {
        Directory dir = newDirectory();
        RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
        final int numDocs = atLeast(100);
        for (int i = 0; i < numDocs; ++i) {
            Document doc = new Document();
            final int numValues = TestUtil.nextInt(random(), 0, maxValuesPerDoc);
            for (int j = 0; j < numValues; ++j) {
                final long value = TestUtil.nextLong(random(), -100, 10000);
                if (sortedNumeric) {
                    doc.add(new SortedNumericDocValuesField("dv", value));
                } else {
                    doc.add(new NumericDocValuesField("dv", value));
                }
                doc.add(new LongPoint("idx", value));
            }
            iw.addDocument(doc);
        }
        if (random().nextBoolean()) {
            iw.deleteDocuments(LongPoint.newRangeQuery("idx", 0L, 10L));
        }
        final IndexReader reader = iw.getReader();
        final IndexSearcher searcher = newSearcher(reader, false);
        iw.close();
        for (int i = 0; i < 100; ++i) {
            final long min = random().nextBoolean() ? Long.MIN_VALUE : TestUtil.nextLong(random(), -100, 10000);
            final long max = random().nextBoolean() ? Long.MAX_VALUE : TestUtil.nextLong(random(), -100, 10000);
            final Query q1 = LongPoint.newRangeQuery("idx", min, max);
            final Query q2;
            if (sortedNumeric) {
                q2 = SortedNumericDocValuesField.newRangeQuery("dv", min, max);
            } else {
                q2 = NumericDocValuesField.newRangeQuery("dv", min, max);
            }
            assertSameMatches(searcher, q1, q2, false);
        }
        reader.close();
        dir.close();
    }
}

Example 63 with NumericDocValuesField

use of org.apache.lucene.document.NumericDocValuesField in project lucene-solr by apache.

the class TestDoubleValuesSource method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    int numDocs = TestUtil.nextInt(random(), 2049, 4000);
    for (int i = 0; i < numDocs; i++) {
        Document document = new Document();
        document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
        document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
        document.add(new NumericDocValuesField("int", random().nextInt()));
        document.add(new NumericDocValuesField("long", random().nextLong()));
        document.add(new FloatDocValuesField("float", random().nextFloat()));
        document.add(new DoubleDocValuesField("double", random().nextDouble()));
        if (i == 545)
            document.add(new DoubleDocValuesField("onefield", 45.72));
        iw.addDocument(document);
    }
    reader = iw.getReader();
    iw.close();
    searcher = newSearcher(reader);
}

Example 64 with NumericDocValuesField

use of org.apache.lucene.document.NumericDocValuesField in project lucene-solr by apache.

the class TestEarlyTerminatingSortingCollector method randomDocument.

private Document randomDocument() {
    final Document doc = new Document();
    doc.add(new NumericDocValuesField("ndv1", random().nextInt(10)));
    doc.add(new NumericDocValuesField("ndv2", random().nextInt(10)));
    doc.add(new StringField("s", RandomPicks.randomFrom(random(), terms), Store.YES));
    return doc;
}

Example 65 with NumericDocValuesField

use of org.apache.lucene.document.NumericDocValuesField in project lucene-solr by apache.

the class TestIndexOrDocValuesQuery method testUseIndexForSelectiveQueries.

public void testUseIndexForSelectiveQueries() throws IOException {
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setCodec(TestUtil.getDefaultCodec()));
    for (int i = 0; i < 2000; ++i) {
        Document doc = new Document();
        if (i == 42) {
            doc.add(new StringField("f1", "bar", Store.NO));
            doc.add(new LongPoint("f2", 42L));
            doc.add(new NumericDocValuesField("f2", 42L));
        } else if (i == 100) {
            doc.add(new StringField("f1", "foo", Store.NO));
            doc.add(new LongPoint("f2", 2L));
            doc.add(new NumericDocValuesField("f2", 2L));
        } else {
            doc.add(new StringField("f1", "bar", Store.NO));
            doc.add(new LongPoint("f2", 2L));
            doc.add(new NumericDocValuesField("f2", 2L));
        }
        w.addDocument(doc);
    }
    w.forceMerge(1);
    IndexReader reader = DirectoryReader.open(w);
    IndexSearcher searcher = newSearcher(reader);
    searcher.setQueryCache(null);
    // The term query is more selective, so the IndexOrDocValuesQuery should use doc values
    final Query q1 = new BooleanQuery.Builder().add(new TermQuery(new Term("f1", "foo")), Occur.MUST).add(new IndexOrDocValuesQuery(LongPoint.newExactQuery("f2", 2), NumericDocValuesField.newRangeQuery("f2", 2L, 2L)), Occur.MUST).build();
    final Weight w1 = searcher.createNormalizedWeight(q1, random().nextBoolean());
    final Scorer s1 = w1.scorer(searcher.getIndexReader().leaves().get(0));
    // means we use doc values
    assertNotNull(s1.twoPhaseIterator());
    // The term query is less selective, so the IndexOrDocValuesQuery should use points
    final Query q2 = new BooleanQuery.Builder().add(new TermQuery(new Term("f1", "bar")), Occur.MUST).add(new IndexOrDocValuesQuery(LongPoint.newExactQuery("f2", 42), NumericDocValuesField.newRangeQuery("f2", 42L, 42L)), Occur.MUST).build();
    final Weight w2 = searcher.createNormalizedWeight(q2, random().nextBoolean());
    final Scorer s2 = w2.scorer(searcher.getIndexReader().leaves().get(0));
    // means we use points
    assertNull(s2.twoPhaseIterator());
    reader.close();
    w.close();
    dir.close();
}