Examples with SortedNumericDocValuesField org.apache.lucene.document.SortedNumericDocValuesField used on opensource projects

Example 41 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class BaseDocValuesFormatTestCase method testThreads2.

/** Tests dv against stored fields with threads (all types + missing) */
@Slow
public void testThreads2() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
    Field idField = new StringField("id", "", Field.Store.NO);
    Field storedBinField = new StoredField("storedBin", new byte[0]);
    Field dvBinField = new BinaryDocValuesField("dvBin", new BytesRef());
    Field dvSortedField = new SortedDocValuesField("dvSorted", new BytesRef());
    Field storedNumericField = new StoredField("storedNum", "");
    Field dvNumericField = new NumericDocValuesField("dvNum", 0);
    // index some docs
    int numDocs = TestUtil.nextInt(random(), 1025, 2047);
    for (int i = 0; i < numDocs; i++) {
        idField.setStringValue(Integer.toString(i));
        int length = TestUtil.nextInt(random(), 0, 8);
        byte[] buffer = new byte[length];
        random().nextBytes(buffer);
        storedBinField.setBytesValue(buffer);
        dvBinField.setBytesValue(buffer);
        dvSortedField.setBytesValue(buffer);
        long numericValue = random().nextLong();
        storedNumericField.setStringValue(Long.toString(numericValue));
        dvNumericField.setLongValue(numericValue);
        Document doc = new Document();
        doc.add(idField);
        if (random().nextInt(4) > 0) {
            doc.add(storedBinField);
            doc.add(dvBinField);
            doc.add(dvSortedField);
        }
        if (random().nextInt(4) > 0) {
            doc.add(storedNumericField);
            doc.add(dvNumericField);
        }
        int numSortedSetFields = random().nextInt(3);
        Set<String> values = new TreeSet<>();
        for (int j = 0; j < numSortedSetFields; j++) {
            values.add(TestUtil.randomSimpleString(random()));
        }
        for (String v : values) {
            doc.add(new SortedSetDocValuesField("dvSortedSet", new BytesRef(v)));
            doc.add(new StoredField("storedSortedSet", v));
        }
        int numSortedNumericFields = random().nextInt(3);
        Set<Long> numValues = new TreeSet<>();
        for (int j = 0; j < numSortedNumericFields; j++) {
            numValues.add(TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE));
        }
        for (Long l : numValues) {
            doc.add(new SortedNumericDocValuesField("dvSortedNumeric", l));
            doc.add(new StoredField("storedSortedNumeric", Long.toString(l)));
        }
        writer.addDocument(doc);
        if (random().nextInt(31) == 0) {
            writer.commit();
        }
    }
    // delete some docs
    int numDeletions = random().nextInt(numDocs / 10);
    for (int i = 0; i < numDeletions; i++) {
        int id = random().nextInt(numDocs);
        writer.deleteDocuments(new Term("id", Integer.toString(id)));
    }
    writer.close();
    // compare
    final DirectoryReader ir = DirectoryReader.open(dir);
    int numThreads = TestUtil.nextInt(random(), 2, 7);
    Thread[] threads = new Thread[numThreads];
    final CountDownLatch startingGun = new CountDownLatch(1);
    for (int i = 0; i < threads.length; i++) {
        threads[i] = new Thread() {

            @Override
            public void run() {
                try {
                    startingGun.await();
                    for (LeafReaderContext context : ir.leaves()) {
                        LeafReader r = context.reader();
                        BinaryDocValues binaries = r.getBinaryDocValues("dvBin");
                        SortedDocValues sorted = r.getSortedDocValues("dvSorted");
                        NumericDocValues numerics = r.getNumericDocValues("dvNum");
                        SortedSetDocValues sortedSet = r.getSortedSetDocValues("dvSortedSet");
                        SortedNumericDocValues sortedNumeric = r.getSortedNumericDocValues("dvSortedNumeric");
                        for (int j = 0; j < r.maxDoc(); j++) {
                            BytesRef binaryValue = r.document(j).getBinaryValue("storedBin");
                            if (binaryValue != null) {
                                if (binaries != null) {
                                    assertEquals(j, binaries.nextDoc());
                                    BytesRef scratch = binaries.binaryValue();
                                    assertEquals(binaryValue, scratch);
                                    assertEquals(j, sorted.nextDoc());
                                    scratch = sorted.binaryValue();
                                    assertEquals(binaryValue, scratch);
                                }
                            }
                            String number = r.document(j).get("storedNum");
                            if (number != null) {
                                if (numerics != null) {
                                    assertEquals(j, numerics.advance(j));
                                    assertEquals(Long.parseLong(number), numerics.longValue());
                                }
                            }
                            String[] values = r.document(j).getValues("storedSortedSet");
                            if (values.length > 0) {
                                assertNotNull(sortedSet);
                                assertEquals(j, sortedSet.nextDoc());
                                for (int k = 0; k < values.length; k++) {
                                    long ord = sortedSet.nextOrd();
                                    assertTrue(ord != SortedSetDocValues.NO_MORE_ORDS);
                                    BytesRef value = sortedSet.lookupOrd(ord);
                                    assertEquals(values[k], value.utf8ToString());
                                }
                                assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
                            }
                            String[] numValues = r.document(j).getValues("storedSortedNumeric");
                            if (numValues.length > 0) {
                                assertNotNull(sortedNumeric);
                                assertEquals(j, sortedNumeric.nextDoc());
                                assertEquals(numValues.length, sortedNumeric.docValueCount());
                                for (int k = 0; k < numValues.length; k++) {
                                    long v = sortedNumeric.nextValue();
                                    assertEquals(numValues[k], Long.toString(v));
                                }
                            }
                        }
                    }
                    TestUtil.checkReader(ir);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        threads[i].start();
    }
    startingGun.countDown();
    for (Thread t : threads) {
        t.join();
    }
    ir.close();
    dir.close();
}

Example 42 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class BaseDocValuesFormatTestCase method testThreads3.

@Slow
public void testThreads3() throws Exception {
    Directory dir = newFSDirectory(createTempDir());
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
    int numSortedSets = random().nextInt(21);
    int numBinaries = random().nextInt(21);
    int numSortedNums = random().nextInt(21);
    int numDocs = TestUtil.nextInt(random(), 2025, 2047);
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        for (int j = 0; j < numSortedSets; j++) {
            doc.add(new SortedSetDocValuesField("ss" + j, new BytesRef(TestUtil.randomSimpleString(random()))));
            doc.add(new SortedSetDocValuesField("ss" + j, new BytesRef(TestUtil.randomSimpleString(random()))));
        }
        for (int j = 0; j < numBinaries; j++) {
            doc.add(new BinaryDocValuesField("b" + j, new BytesRef(TestUtil.randomSimpleString(random()))));
        }
        for (int j = 0; j < numSortedNums; j++) {
            doc.add(new SortedNumericDocValuesField("sn" + j, TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
            doc.add(new SortedNumericDocValuesField("sn" + j, TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
        }
        writer.addDocument(doc);
    }
    writer.close();
    // now check with threads
    for (int i = 0; i < 10; i++) {
        final DirectoryReader r = DirectoryReader.open(dir);
        final CountDownLatch startingGun = new CountDownLatch(1);
        Thread[] threads = new Thread[TestUtil.nextInt(random(), 4, 10)];
        for (int tid = 0; tid < threads.length; tid++) {
            threads[tid] = new Thread() {

                @Override
                public void run() {
                    try {
                        ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
                        PrintStream infoStream = new PrintStream(bos, false, IOUtils.UTF_8);
                        startingGun.await();
                        for (LeafReaderContext leaf : r.leaves()) {
                            DocValuesStatus status = CheckIndex.testDocValues((SegmentReader) leaf.reader(), infoStream, true);
                            if (status.error != null) {
                                throw status.error;
                            }
                        }
                    } catch (Throwable e) {
                        throw new RuntimeException(e);
                    }
                }
            };
        }
        for (int tid = 0; tid < threads.length; tid++) {
            threads[tid].start();
        }
        startingGun.countDown();
        for (int tid = 0; tid < threads.length; tid++) {
            threads[tid].join();
        }
        r.close();
    }
    dir.close();
}

Example 43 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class BaseDocValuesFormatTestCase method testOneSortedNumberOneMissing.

public void testOneSortedNumberOneMissing() throws IOException {
    Directory directory = newDirectory();
    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(null));
    Document doc = new Document();
    doc.add(new SortedNumericDocValuesField("dv", 5));
    writer.addDocument(doc);
    writer.addDocument(new Document());
    writer.close();
    // Now search the index:
    IndexReader reader = DirectoryReader.open(directory);
    assert reader.leaves().size() == 1;
    SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
    assertEquals(0, dv.nextDoc());
    assertEquals(1, dv.docValueCount());
    assertEquals(5, dv.nextValue());
    assertEquals(NO_MORE_DOCS, dv.nextDoc());
    reader.close();
    directory.close();
}

Example 44 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class BaseDocValuesFormatTestCase method testTwoSortedNumberSameValue.

public void testTwoSortedNumberSameValue() throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
    Document doc = new Document();
    doc.add(new SortedNumericDocValuesField("dv", 11));
    doc.add(new SortedNumericDocValuesField("dv", 11));
    writer.addDocument(doc);
    writer.close();
    // Now search the index:
    IndexReader reader = DirectoryReader.open(directory);
    assert reader.leaves().size() == 1;
    SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
    assertEquals(0, dv.nextDoc());
    assertEquals(2, dv.docValueCount());
    assertEquals(11, dv.nextValue());
    assertEquals(11, dv.nextValue());
    reader.close();
    directory.close();
}

Example 45 with SortedNumericDocValuesField

use of org.apache.lucene.document.SortedNumericDocValuesField in project lucene-solr by apache.

the class TestValueSources method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    dir = newDirectory();
    analyzer = new MockAnalyzer(random());
    IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer);
    iwConfig.setMergePolicy(newLogMergePolicy());
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);
    for (String[] doc : documents) {
        Document document = new Document();
        document.add(new StringField("id", doc[0], Field.Store.NO));
        document.add(new SortedDocValuesField("id", new BytesRef(doc[0])));
        document.add(new NumericDocValuesField("double", Double.doubleToRawLongBits(Double.parseDouble(doc[1]))));
        document.add(new NumericDocValuesField("float", Float.floatToRawIntBits(Float.parseFloat(doc[2]))));
        document.add(new NumericDocValuesField("int", Integer.parseInt(doc[3])));
        document.add(new NumericDocValuesField("long", Long.parseLong(doc[4])));
        document.add(new StringField("string", doc[5], Field.Store.NO));
        document.add(new SortedDocValuesField("string", new BytesRef(doc[5])));
        document.add(new TextField("text", doc[6], Field.Store.NO));
        document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[7]))));
        document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[8]))));
        document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[9]))));
        document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[7]))));
        document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[8]))));
        document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[9]))));
        document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[10])));
        document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[11])));
        document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[12])));
        document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[10])));
        document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[11])));
        document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[12])));
        iw.addDocument(document);
    }
    reader = iw.getReader();
    searcher = newSearcher(reader);
    iw.close();
}