Examples with NumericDocValues org.apache.lucene.index.NumericDocValues used on opensource projects

Example 21 with NumericDocValues

use of org.apache.lucene.index.NumericDocValues in project lucene-solr by apache.

the class TestFieldCache method testGetDocsWithFieldThreadSafety.

public void testGetDocsWithFieldThreadSafety() throws Exception {
    final FieldCache cache = FieldCache.DEFAULT;
    cache.purgeAllCaches();
    int NUM_THREADS = 3;
    Thread[] threads = new Thread[NUM_THREADS];
    final AtomicBoolean failed = new AtomicBoolean();
    final AtomicInteger iters = new AtomicInteger();
    final int NUM_ITER = 200 * RANDOM_MULTIPLIER;
    final CyclicBarrier restart = new CyclicBarrier(NUM_THREADS, new Runnable() {

        @Override
        public void run() {
            cache.purgeAllCaches();
            iters.incrementAndGet();
        }
    });
    for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++) {
        threads[threadIDX] = new Thread() {

            @Override
            public void run() {
                try {
                    while (!failed.get()) {
                        final int op = random().nextInt(3);
                        if (op == 0) {
                            // Purge all caches & resume, once all
                            // threads get here:
                            restart.await();
                            if (iters.get() >= NUM_ITER) {
                                break;
                            }
                        } else if (op == 1) {
                            Bits docsWithField = cache.getDocsWithField(reader, "sparse", FieldCache.INT_POINT_PARSER);
                            for (int i = 0; i < docsWithField.length(); i++) {
                                assertEquals(i % 2 == 0, docsWithField.get(i));
                            }
                        } else {
                            NumericDocValues ints = cache.getNumerics(reader, "sparse", FieldCache.INT_POINT_PARSER);
                            for (int i = 0; i < reader.maxDoc(); i++) {
                                if (i % 2 == 0) {
                                    assertEquals(i, ints.nextDoc());
                                    assertEquals(i, ints.longValue());
                                }
                            }
                        }
                    }
                } catch (Throwable t) {
                    failed.set(true);
                    restart.reset();
                    throw new RuntimeException(t);
                }
            }
        };
        threads[threadIDX].start();
    }
    for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++) {
        threads[threadIDX].join();
    }
    assertFalse(failed.get());
}

Example 22 with NumericDocValues

use of org.apache.lucene.index.NumericDocValues in project lucene-solr by apache.

the class TestFieldCache method testDocValuesIntegration.

public void testDocValuesIntegration() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(null);
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
    Document doc = new Document();
    doc.add(new BinaryDocValuesField("binary", new BytesRef("binary value")));
    doc.add(new SortedDocValuesField("sorted", new BytesRef("sorted value")));
    doc.add(new NumericDocValuesField("numeric", 42));
    doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1")));
    doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2")));
    iw.addDocument(doc);
    DirectoryReader ir = iw.getReader();
    iw.close();
    LeafReader ar = getOnlyLeafReader(ir);
    // Binary type: can be retrieved via getTerms()
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.INT_POINT_PARSER);
    });
    BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary");
    assertEquals(0, binary.nextDoc());
    final BytesRef term = binary.binaryValue();
    assertEquals("binary value", term.utf8ToString());
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTermsIndex(ar, "binary");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getDocTermOrds(ar, "binary", null);
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "binary");
    });
    Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary", null);
    assertTrue(bits.get(0));
    // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.INT_POINT_PARSER);
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "sorted");
    });
    binary = FieldCache.DEFAULT.getTerms(ar, "sorted");
    assertEquals(0, binary.nextDoc());
    BytesRef scratch = binary.binaryValue();
    assertEquals("sorted value", scratch.utf8ToString());
    SortedDocValues sorted = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
    assertEquals(0, sorted.nextDoc());
    assertEquals(0, sorted.ordValue());
    assertEquals(1, sorted.getValueCount());
    scratch = sorted.binaryValue();
    assertEquals("sorted value", scratch.utf8ToString());
    SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted", null);
    assertEquals(0, sortedSet.nextDoc());
    assertEquals(0, sortedSet.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    assertEquals(1, sortedSet.getValueCount());
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted", null);
    assertTrue(bits.get(0));
    // Numeric type: can be retrieved via getInts() and so on
    NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.INT_POINT_PARSER);
    assertEquals(0, numeric.nextDoc());
    assertEquals(42, numeric.longValue());
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTerms(ar, "numeric");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTermsIndex(ar, "numeric");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getDocTermOrds(ar, "numeric", null);
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "numeric");
    });
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric", null);
    assertTrue(bits.get(0));
    // SortedSet type: can be retrieved via getDocTermOrds() 
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.INT_POINT_PARSER);
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTerms(ar, "sortedset");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTermsIndex(ar, "sortedset");
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "sortedset");
    });
    sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sortedset", null);
    assertEquals(0, sortedSet.nextDoc());
    assertEquals(0, sortedSet.nextOrd());
    assertEquals(1, sortedSet.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    assertEquals(2, sortedSet.getValueCount());
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset", null);
    assertTrue(bits.get(0));
    ir.close();
    dir.close();
}

Example 23 with NumericDocValues

use of org.apache.lucene.index.NumericDocValues in project lucene-solr by apache.

the class TestFieldCache method testIntFieldCache.

// Make sure that the use of GrowableWriter doesn't prevent from using the full int range
public void testIntFieldCache() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random()));
    cfg.setMergePolicy(newLogMergePolicy());
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, cfg);
    Document doc = new Document();
    IntPoint field = new IntPoint("f", 0);
    doc.add(field);
    final int[] values = new int[TestUtil.nextInt(random(), 1, 10)];
    Set<Integer> missing = new HashSet<>();
    for (int i = 0; i < values.length; ++i) {
        final int v;
        switch(random().nextInt(10)) {
            case 0:
                v = Integer.MIN_VALUE;
                break;
            case 1:
                v = 0;
                break;
            case 2:
                v = Integer.MAX_VALUE;
                break;
            default:
                v = TestUtil.nextInt(random(), -10, 10);
                break;
        }
        values[i] = v;
        if (v == 0 && random().nextBoolean()) {
            // missing
            iw.addDocument(new Document());
            missing.add(i);
        } else {
            field.setIntValue(v);
            iw.addDocument(doc);
        }
    }
    iw.forceMerge(1);
    final DirectoryReader reader = iw.getReader();
    final NumericDocValues ints = FieldCache.DEFAULT.getNumerics(getOnlyLeafReader(reader), "f", FieldCache.INT_POINT_PARSER);
    for (int i = 0; i < values.length; ++i) {
        if (missing.contains(i) == false) {
            assertEquals(i, ints.nextDoc());
            assertEquals(values[i], ints.longValue());
        }
    }
    assertEquals(NO_MORE_DOCS, ints.nextDoc());
    reader.close();
    iw.close();
    dir.close();
}

Example 24 with NumericDocValues

use of org.apache.lucene.index.NumericDocValues in project lucene-solr by apache.

the class TestFieldCacheWithThreads method test2.

public void test2() throws Exception {
    Random random = random();
    final int NUM_DOCS = atLeast(100);
    final Directory dir = newDirectory();
    final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
    final boolean allowDups = random.nextBoolean();
    final Set<String> seen = new HashSet<>();
    if (VERBOSE) {
        System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " allowDups=" + allowDups);
    }
    int numDocs = 0;
    final List<BytesRef> docValues = new ArrayList<>();
    // TODO: deletions
    while (numDocs < NUM_DOCS) {
        final String s;
        if (random.nextBoolean()) {
            s = TestUtil.randomSimpleString(random);
        } else {
            s = TestUtil.randomUnicodeString(random);
        }
        final BytesRef br = new BytesRef(s);
        if (!allowDups) {
            if (seen.contains(s)) {
                continue;
            }
            seen.add(s);
        }
        if (VERBOSE) {
            System.out.println("  " + numDocs + ": s=" + s);
        }
        final Document doc = new Document();
        doc.add(new SortedDocValuesField("stringdv", br));
        doc.add(new NumericDocValuesField("id", numDocs));
        docValues.add(br);
        writer.addDocument(doc);
        numDocs++;
        if (random.nextInt(40) == 17) {
            // force flush
            writer.getReader().close();
        }
    }
    writer.forceMerge(1);
    final DirectoryReader r = writer.getReader();
    writer.close();
    final LeafReader sr = getOnlyLeafReader(r);
    final long END_TIME = System.nanoTime() + TimeUnit.NANOSECONDS.convert((TEST_NIGHTLY ? 30 : 1), TimeUnit.SECONDS);
    final int NUM_THREADS = TestUtil.nextInt(random(), 1, 10);
    Thread[] threads = new Thread[NUM_THREADS];
    for (int thread = 0; thread < NUM_THREADS; thread++) {
        threads[thread] = new Thread() {

            @Override
            public void run() {
                Random random = random();
                final SortedDocValues stringDVDirect;
                final NumericDocValues docIDToID;
                try {
                    stringDVDirect = sr.getSortedDocValues("stringdv");
                    docIDToID = sr.getNumericDocValues("id");
                    assertNotNull(stringDVDirect);
                } catch (IOException ioe) {
                    throw new RuntimeException(ioe);
                }
                int[] docIDToIDArray = new int[sr.maxDoc()];
                for (int i = 0; i < sr.maxDoc(); i++) {
                    try {
                        assertEquals(i, docIDToID.nextDoc());
                    } catch (IOException ioe) {
                        throw new RuntimeException(ioe);
                    }
                    try {
                        docIDToIDArray[i] = (int) docIDToID.longValue();
                    } catch (IOException ioe) {
                        throw new RuntimeException(ioe);
                    }
                }
                while (System.nanoTime() < END_TIME) {
                    for (int iter = 0; iter < 100; iter++) {
                        final int docID = random.nextInt(sr.maxDoc());
                        try {
                            SortedDocValues dvs = sr.getSortedDocValues("stringdv");
                            assertEquals(docID, dvs.advance(docID));
                            assertEquals(docValues.get(docIDToIDArray[docID]), dvs.binaryValue());
                        } catch (IOException ioe) {
                            throw new RuntimeException(ioe);
                        }
                    }
                }
            }
        };
        threads[thread].start();
    }
    for (Thread thread : threads) {
        thread.join();
    }
    r.close();
    dir.close();
}

Example 25 with NumericDocValues

use of org.apache.lucene.index.NumericDocValues in project lucene-solr by apache.

the class DocValuesTest method testDocValues.

@Test
public void testDocValues() throws IOException {
    assertU(adoc("id", "1"));
    assertU(commit());
    try (SolrCore core = h.getCoreInc()) {
        final RefCounted<SolrIndexSearcher> searcherRef = core.openNewSearcher(true, true);
        final SolrIndexSearcher searcher = searcherRef.get();
        try {
            final LeafReader reader = searcher.getSlowAtomicReader();
            assertEquals(1, reader.numDocs());
            final FieldInfos infos = reader.getFieldInfos();
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("floatdv").getDocValuesType());
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("intdv").getDocValuesType());
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("doubledv").getDocValuesType());
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("longdv").getDocValuesType());
            assertEquals(DocValuesType.SORTED, infos.fieldInfo("stringdv").getDocValuesType());
            assertEquals(DocValuesType.SORTED, infos.fieldInfo("booldv").getDocValuesType());
            NumericDocValues dvs = reader.getNumericDocValues("floatdv");
            assertEquals(0, dvs.nextDoc());
            assertEquals((long) Float.floatToIntBits(1), dvs.longValue());
            dvs = reader.getNumericDocValues("intdv");
            assertEquals(0, dvs.nextDoc());
            assertEquals(2L, dvs.longValue());
            dvs = reader.getNumericDocValues("doubledv");
            assertEquals(0, dvs.nextDoc());
            assertEquals(Double.doubleToLongBits(3), dvs.longValue());
            dvs = reader.getNumericDocValues("longdv");
            assertEquals(0, dvs.nextDoc());
            assertEquals(4L, dvs.longValue());
            SortedDocValues sdv = reader.getSortedDocValues("stringdv");
            assertEquals(0, sdv.nextDoc());
            assertEquals("solr", sdv.binaryValue().utf8ToString());
            sdv = reader.getSortedDocValues("booldv");
            assertEquals(0, sdv.nextDoc());
            assertEquals("T", sdv.binaryValue().utf8ToString());
            final IndexSchema schema = core.getLatestSchema();
            final SchemaField floatDv = schema.getField("floatdv");
            final SchemaField intDv = schema.getField("intdv");
            final SchemaField doubleDv = schema.getField("doubledv");
            final SchemaField longDv = schema.getField("longdv");
            final SchemaField boolDv = schema.getField("booldv");
            FunctionValues values = floatDv.getType().getValueSource(floatDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(1f, values.floatVal(0), 0f);
            assertEquals(1f, values.objectVal(0));
            values = intDv.getType().getValueSource(intDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(2, values.intVal(0));
            assertEquals(2, values.objectVal(0));
            values = doubleDv.getType().getValueSource(doubleDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(3d, values.doubleVal(0), 0d);
            assertEquals(3d, values.objectVal(0));
            values = longDv.getType().getValueSource(longDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(4L, values.longVal(0));
            assertEquals(4L, values.objectVal(0));
            values = boolDv.getType().getValueSource(boolDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals("true", values.strVal(0));
            assertEquals(true, values.objectVal(0));
            // check reversibility of created fields
            tstToObj(schema.getField("floatdv"), -1.5f);
            tstToObj(schema.getField("floatdvs"), -1.5f);
            tstToObj(schema.getField("doubledv"), -1.5d);
            tstToObj(schema.getField("doubledvs"), -1.5d);
            tstToObj(schema.getField("intdv"), -7);
            tstToObj(schema.getField("intdvs"), -7);
            tstToObj(schema.getField("longdv"), -11L);
            tstToObj(schema.getField("longdvs"), -11L);
            tstToObj(schema.getField("datedv"), new Date(1000));
            tstToObj(schema.getField("datedvs"), new Date(1000));
            tstToObj(schema.getField("stringdv"), "foo");
            tstToObj(schema.getField("stringdvs"), "foo");
            tstToObj(schema.getField("booldv"), true);
            tstToObj(schema.getField("booldvs"), true);
        } finally {
            searcherRef.decref();
        }
    }
}