Examples with SortedNumericDocValuesField org.apache.lucene.document.SortedNumericDocValuesField used on opensource projects

Example 96 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestFieldValueQuery method testRandom.

public void testRandom() throws IOException {
    final int iters = atLeast(10);
    for (int iter = 0; iter < iters; ++iter) {
        Directory dir = newDirectory();
        RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
        final int numDocs = atLeast(100);
        for (int i = 0; i < numDocs; ++i) {
            Document doc = new Document();
            final boolean hasValue = random().nextBoolean();
            if (hasValue) {
                doc.add(new NumericDocValuesField("dv1", 1));
                doc.add(new SortedNumericDocValuesField("dv2", 1));
                doc.add(new SortedNumericDocValuesField("dv2", 2));
                doc.add(new StringField("has_value", "yes", Store.NO));
            }
            doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
            iw.addDocument(doc);
        }
        if (random().nextBoolean()) {
            iw.deleteDocuments(new TermQuery(new Term("f", "no")));
        }
        iw.commit();
        final IndexReader reader = iw.getReader();
        final IndexSearcher searcher = newSearcher(reader);
        iw.close();
        assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new FieldValueQuery("dv1"), false);
        assertSameMatches(searcher, new TermQuery(new Term("has_value", "yes")), new FieldValueQuery("dv2"), false);
        reader.close();
        dir.close();
    }
}

Example 97 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestFieldValueQuery method testScore.

public void testScore() throws IOException {
    final int iters = atLeast(10);
    for (int iter = 0; iter < iters; ++iter) {
        Directory dir = newDirectory();
        RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
        final int numDocs = atLeast(100);
        for (int i = 0; i < numDocs; ++i) {
            Document doc = new Document();
            final boolean hasValue = random().nextBoolean();
            if (hasValue) {
                doc.add(new NumericDocValuesField("dv1", 1));
                doc.add(new SortedNumericDocValuesField("dv2", 1));
                doc.add(new SortedNumericDocValuesField("dv2", 2));
                doc.add(new StringField("has_value", "yes", Store.NO));
            }
            doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
            iw.addDocument(doc);
        }
        if (random().nextBoolean()) {
            iw.deleteDocuments(new TermQuery(new Term("f", "no")));
        }
        iw.commit();
        final IndexReader reader = iw.getReader();
        final IndexSearcher searcher = newSearcher(reader);
        iw.close();
        final float boost = random().nextFloat() * 10;
        final Query ref = new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("has_value", "yes"))), boost);
        final Query q1 = new BoostQuery(new FieldValueQuery("dv1"), boost);
        assertSameMatches(searcher, ref, q1, true);
        final Query q2 = new BoostQuery(new FieldValueQuery("dv2"), boost);
        assertSameMatches(searcher, ref, q2, true);
        reader.close();
        dir.close();
    }
}

Example 98 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestLucene70DocValuesFormat method testSortedNumericAroundBlockSize.

@Slow
public void testSortedNumericAroundBlockSize() throws IOException {
    final int frontier = 1 << Lucene70DocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT;
    for (int maxDoc = frontier - 1; maxDoc <= frontier + 1; ++maxDoc) {
        final Directory dir = newDirectory();
        IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
        RAMFile buffer = new RAMFile();
        RAMOutputStream out = new RAMOutputStream(buffer, false);
        Document doc = new Document();
        SortedNumericDocValuesField field1 = new SortedNumericDocValuesField("snum", 0L);
        doc.add(field1);
        SortedNumericDocValuesField field2 = new SortedNumericDocValuesField("snum", 0L);
        doc.add(field2);
        for (int i = 0; i < maxDoc; ++i) {
            long s1 = random().nextInt(100);
            long s2 = random().nextInt(100);
            field1.setLongValue(s1);
            field2.setLongValue(s2);
            w.addDocument(doc);
            out.writeVLong(Math.min(s1, s2));
            out.writeVLong(Math.max(s1, s2));
        }
        out.close();
        w.forceMerge(1);
        DirectoryReader r = DirectoryReader.open(w);
        w.close();
        LeafReader sr = getOnlyLeafReader(r);
        assertEquals(maxDoc, sr.maxDoc());
        SortedNumericDocValues values = sr.getSortedNumericDocValues("snum");
        assertNotNull(values);
        RAMInputStream in = new RAMInputStream("", buffer);
        for (int i = 0; i < maxDoc; ++i) {
            assertEquals(i, values.nextDoc());
            assertEquals(2, values.docValueCount());
            assertEquals(in.readVLong(), values.nextValue());
            assertEquals(in.readVLong(), values.nextValue());
        }
        r.close();
        dir.close();
    }
}

Example 99 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestLucene70DocValuesFormat method doTestSortedNumericBlocksOfVariousBitsPerValue.

private void doTestSortedNumericBlocksOfVariousBitsPerValue(LongSupplier counts) throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    conf.setMaxBufferedDocs(atLeast(Lucene70DocValuesFormat.NUMERIC_BLOCK_SIZE));
    conf.setRAMBufferSizeMB(-1);
    conf.setMergePolicy(newLogMergePolicy(random().nextBoolean()));
    IndexWriter writer = new IndexWriter(dir, conf);
    final int numDocs = atLeast(Lucene70DocValuesFormat.NUMERIC_BLOCK_SIZE * 3);
    final LongSupplier values = blocksOfVariousBPV();
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        int valueCount = (int) counts.getAsLong();
        long[] valueArray = new long[valueCount];
        for (int j = 0; j < valueCount; j++) {
            long value = values.getAsLong();
            valueArray[j] = value;
            doc.add(new SortedNumericDocValuesField("dv", value));
        }
        Arrays.sort(valueArray);
        for (int j = 0; j < valueCount; j++) {
            doc.add(new StoredField("stored", Long.toString(valueArray[j])));
        }
        writer.addDocument(doc);
        if (random().nextInt(31) == 0) {
            writer.commit();
        }
    }
    writer.forceMerge(1);
    writer.close();
    // compare
    DirectoryReader ir = DirectoryReader.open(dir);
    TestUtil.checkReader(ir);
    for (LeafReaderContext context : ir.leaves()) {
        LeafReader r = context.reader();
        SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");
        for (int i = 0; i < r.maxDoc(); i++) {
            if (i > docValues.docID()) {
                docValues.nextDoc();
            }
            String[] expected = r.document(i).getValues("stored");
            if (i < docValues.docID()) {
                assertEquals(0, expected.length);
            } else {
                String[] actual = new String[docValues.docValueCount()];
                for (int j = 0; j < actual.length; j++) {
                    actual[j] = Long.toString(docValues.nextValue());
                }
                assertArrayEquals(expected, actual);
            }
        }
    }
    ir.close();
    dir.close();
}

Example 100 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestMultiDocValues method testSortedNumeric.

public void testSortedNumeric() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(random(), null);
    iwc.setMergePolicy(newLogMergePolicy());
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
    int numDocs = TEST_NIGHTLY ? atLeast(500) : atLeast(50);
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        int numValues = random().nextInt(5);
        for (int j = 0; j < numValues; j++) {
            doc.add(new SortedNumericDocValuesField("nums", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
        }
        iw.addDocument(doc);
        if (random().nextInt(17) == 0) {
            iw.commit();
        }
    }
    DirectoryReader ir = iw.getReader();
    iw.forceMerge(1);
    DirectoryReader ir2 = iw.getReader();
    LeafReader merged = getOnlyLeafReader(ir2);
    iw.close();
    SortedNumericDocValues multi = MultiDocValues.getSortedNumericValues(ir, "nums");
    SortedNumericDocValues single = merged.getSortedNumericDocValues("nums");
    if (multi == null) {
        assertNull(single);
    } else {
        // check values
        for (int i = 0; i < numDocs; i++) {
            if (i > single.docID()) {
                assertEquals(single.nextDoc(), multi.nextDoc());
            }
            if (i == single.docID()) {
                assertEquals(single.docValueCount(), multi.docValueCount());
                for (int j = 0; j < single.docValueCount(); j++) {
                    assertEquals(single.nextValue(), multi.nextValue());
                }
            }
        }
    }
    testRandomAdvance(merged.getSortedNumericDocValues("nums"), MultiDocValues.getSortedNumericValues(ir, "nums"));
    testRandomAdvanceExact(merged.getSortedNumericDocValues("nums"), MultiDocValues.getSortedNumericValues(ir, "nums"), merged.maxDoc());
    ir.close();
    ir2.close();
    dir.close();
}