Examples with LeafReader org.apache.lucene.index.LeafReader used on opensource projects

Example 56 with LeafReader

use of org.apache.lucene.index.LeafReader in project lucene-solr by apache.

the class TestFieldCache method testEmptyIndex.

public void testEmptyIndex() throws Exception {
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(500));
    writer.close();
    IndexReader r = DirectoryReader.open(dir);
    LeafReader reader = SlowCompositeReaderWrapper.wrap(r);
    TestUtil.checkReader(reader);
    FieldCache.DEFAULT.getTerms(reader, "foobar");
    FieldCache.DEFAULT.getTermsIndex(reader, "foobar");
    FieldCache.DEFAULT.purgeByCacheKey(reader.getCoreCacheHelper().getKey());
    r.close();
    dir.close();
}

Example 57 with LeafReader

use of org.apache.lucene.index.LeafReader in project lucene-solr by apache.

the class TestFieldCache method testDocValuesIntegration.

public void testDocValuesIntegration() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(null);
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
    Document doc = new Document();
    doc.add(new BinaryDocValuesField("binary", new BytesRef("binary value")));
    doc.add(new SortedDocValuesField("sorted", new BytesRef("sorted value")));
    doc.add(new NumericDocValuesField("numeric", 42));
    doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1")));
    doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2")));
    iw.addDocument(doc);
    DirectoryReader ir = iw.getReader();
    iw.close();
    LeafReader ar = getOnlyLeafReader(ir);
    // Binary type: can be retrieved via getTerms()
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.INT_POINT_PARSER);
    });
    BinaryDocValues binary = FieldCache.DEFAULT.getTerms(ar, "binary");
    assertEquals(0, binary.nextDoc());
    final BytesRef term = binary.binaryValue();
    assertEquals("binary value", term.utf8ToString());
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTermsIndex(ar, "binary");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getDocTermOrds(ar, "binary", null);
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "binary");
    });
    Bits bits = FieldCache.DEFAULT.getDocsWithField(ar, "binary", null);
    assertTrue(bits.get(0));
    // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.INT_POINT_PARSER);
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "sorted");
    });
    binary = FieldCache.DEFAULT.getTerms(ar, "sorted");
    assertEquals(0, binary.nextDoc());
    BytesRef scratch = binary.binaryValue();
    assertEquals("sorted value", scratch.utf8ToString());
    SortedDocValues sorted = FieldCache.DEFAULT.getTermsIndex(ar, "sorted");
    assertEquals(0, sorted.nextDoc());
    assertEquals(0, sorted.ordValue());
    assertEquals(1, sorted.getValueCount());
    scratch = sorted.binaryValue();
    assertEquals("sorted value", scratch.utf8ToString());
    SortedSetDocValues sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sorted", null);
    assertEquals(0, sortedSet.nextDoc());
    assertEquals(0, sortedSet.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    assertEquals(1, sortedSet.getValueCount());
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "sorted", null);
    assertTrue(bits.get(0));
    // Numeric type: can be retrieved via getInts() and so on
    NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.INT_POINT_PARSER);
    assertEquals(0, numeric.nextDoc());
    assertEquals(42, numeric.longValue());
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTerms(ar, "numeric");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTermsIndex(ar, "numeric");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getDocTermOrds(ar, "numeric", null);
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "numeric");
    });
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "numeric", null);
    assertTrue(bits.get(0));
    // SortedSet type: can be retrieved via getDocTermOrds() 
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.INT_POINT_PARSER);
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTerms(ar, "sortedset");
    });
    expectThrows(IllegalStateException.class, () -> {
        FieldCache.DEFAULT.getTermsIndex(ar, "sortedset");
    });
    expectThrows(IllegalStateException.class, () -> {
        new DocTermOrds(ar, null, "sortedset");
    });
    sortedSet = FieldCache.DEFAULT.getDocTermOrds(ar, "sortedset", null);
    assertEquals(0, sortedSet.nextDoc());
    assertEquals(0, sortedSet.nextOrd());
    assertEquals(1, sortedSet.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSet.nextOrd());
    assertEquals(2, sortedSet.getValueCount());
    bits = FieldCache.DEFAULT.getDocsWithField(ar, "sortedset", null);
    assertTrue(bits.get(0));
    ir.close();
    dir.close();
}

Example 58 with LeafReader

use of org.apache.lucene.index.LeafReader in project lucene-solr by apache.

the class TestFieldCacheWithThreads method test2.

public void test2() throws Exception {
    Random random = random();
    final int NUM_DOCS = atLeast(100);
    final Directory dir = newDirectory();
    final RandomIndexWriter writer = new RandomIndexWriter(random, dir);
    final boolean allowDups = random.nextBoolean();
    final Set<String> seen = new HashSet<>();
    if (VERBOSE) {
        System.out.println("TEST: NUM_DOCS=" + NUM_DOCS + " allowDups=" + allowDups);
    }
    int numDocs = 0;
    final List<BytesRef> docValues = new ArrayList<>();
    // TODO: deletions
    while (numDocs < NUM_DOCS) {
        final String s;
        if (random.nextBoolean()) {
            s = TestUtil.randomSimpleString(random);
        } else {
            s = TestUtil.randomUnicodeString(random);
        }
        final BytesRef br = new BytesRef(s);
        if (!allowDups) {
            if (seen.contains(s)) {
                continue;
            }
            seen.add(s);
        }
        if (VERBOSE) {
            System.out.println("  " + numDocs + ": s=" + s);
        }
        final Document doc = new Document();
        doc.add(new SortedDocValuesField("stringdv", br));
        doc.add(new NumericDocValuesField("id", numDocs));
        docValues.add(br);
        writer.addDocument(doc);
        numDocs++;
        if (random.nextInt(40) == 17) {
            // force flush
            writer.getReader().close();
        }
    }
    writer.forceMerge(1);
    final DirectoryReader r = writer.getReader();
    writer.close();
    final LeafReader sr = getOnlyLeafReader(r);
    final long END_TIME = System.nanoTime() + TimeUnit.NANOSECONDS.convert((TEST_NIGHTLY ? 30 : 1), TimeUnit.SECONDS);
    final int NUM_THREADS = TestUtil.nextInt(random(), 1, 10);
    Thread[] threads = new Thread[NUM_THREADS];
    for (int thread = 0; thread < NUM_THREADS; thread++) {
        threads[thread] = new Thread() {

            @Override
            public void run() {
                Random random = random();
                final SortedDocValues stringDVDirect;
                final NumericDocValues docIDToID;
                try {
                    stringDVDirect = sr.getSortedDocValues("stringdv");
                    docIDToID = sr.getNumericDocValues("id");
                    assertNotNull(stringDVDirect);
                } catch (IOException ioe) {
                    throw new RuntimeException(ioe);
                }
                int[] docIDToIDArray = new int[sr.maxDoc()];
                for (int i = 0; i < sr.maxDoc(); i++) {
                    try {
                        assertEquals(i, docIDToID.nextDoc());
                    } catch (IOException ioe) {
                        throw new RuntimeException(ioe);
                    }
                    try {
                        docIDToIDArray[i] = (int) docIDToID.longValue();
                    } catch (IOException ioe) {
                        throw new RuntimeException(ioe);
                    }
                }
                while (System.nanoTime() < END_TIME) {
                    for (int iter = 0; iter < 100; iter++) {
                        final int docID = random.nextInt(sr.maxDoc());
                        try {
                            SortedDocValues dvs = sr.getSortedDocValues("stringdv");
                            assertEquals(docID, dvs.advance(docID));
                            assertEquals(docValues.get(docIDToIDArray[docID]), dvs.binaryValue());
                        } catch (IOException ioe) {
                            throw new RuntimeException(ioe);
                        }
                    }
                }
            }
        };
        threads[thread].start();
    }
    for (Thread thread : threads) {
        thread.join();
    }
    r.close();
    dir.close();
}

Example 59 with LeafReader

use of org.apache.lucene.index.LeafReader in project lucene-solr by apache.

the class TestSlowCompositeReaderWrapper method testCoreListenerOnSlowCompositeReaderWrapper.

public void testCoreListenerOnSlowCompositeReaderWrapper() throws IOException {
    RandomIndexWriter w = new RandomIndexWriter(random(), newDirectory());
    final int numDocs = TestUtil.nextInt(random(), 1, 5);
    for (int i = 0; i < numDocs; ++i) {
        w.addDocument(new Document());
        if (random().nextBoolean()) {
            w.commit();
        }
    }
    w.commit();
    w.close();
    final IndexReader reader = DirectoryReader.open(w.w.getDirectory());
    final LeafReader leafReader = SlowCompositeReaderWrapper.wrap(reader);
    final int numListeners = TestUtil.nextInt(random(), 1, 10);
    final List<IndexReader.ClosedListener> listeners = new ArrayList<>();
    AtomicInteger counter = new AtomicInteger(numListeners);
    for (int i = 0; i < numListeners; ++i) {
        CountCoreListener listener = new CountCoreListener(counter, leafReader.getCoreCacheHelper().getKey());
        listeners.add(listener);
        leafReader.getCoreCacheHelper().addClosedListener(listener);
    }
    for (int i = 0; i < 100; ++i) {
        leafReader.getCoreCacheHelper().addClosedListener(listeners.get(random().nextInt(listeners.size())));
    }
    assertEquals(numListeners, counter.get());
    // make sure listeners are registered on the wrapped reader and that closing any of them has the same effect
    if (random().nextBoolean()) {
        reader.close();
    } else {
        leafReader.close();
    }
    assertEquals(0, counter.get());
    w.w.getDirectory().close();
}

Example 60 with LeafReader

use of org.apache.lucene.index.LeafReader in project lucene-solr by apache.

the class UninvertDocValuesMergePolicyTest method testNonIndexedFieldDoesNonFail.

// When an non-indexed field gets merged, it exhibit the old behavior
// The field will be merged, docvalues headers updated, but no docvalues for this field
public void testNonIndexedFieldDoesNonFail() throws Exception {
    // Remove Indexed from fieldType
    removeIndexFrom(h, TEST_FIELD);
    assertU(adoc(ID_FIELD, String.valueOf(1), TEST_FIELD, String.valueOf(1)));
    assertU(commit());
    addDocValuesTo(h, TEST_FIELD);
    assertU(adoc(ID_FIELD, String.valueOf(2), TEST_FIELD, String.valueOf(2)));
    assertU(commit());
    assertU(optimize("maxSegments", "1"));
    withNewRawReader(h, topReader -> {
        assertEquals(2, topReader.numDocs());
        assertEquals(1, topReader.leaves().size());
        final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader);
        assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType());
        for (LeafReaderContext ctx : topReader.leaves()) {
            LeafReader r = ctx.reader();
            SortedDocValues docvalues = r.getSortedDocValues(TEST_FIELD);
            for (int i = 0; i < r.numDocs(); ++i) {
                Document doc = r.document(i);
                String v = doc.getField(TEST_FIELD).stringValue();
                String id = doc.getField(ID_FIELD).stringValue();
                assertEquals(DocValuesType.SORTED, r.getFieldInfos().fieldInfo(TEST_FIELD).getDocValuesType());
                assertEquals(DocValuesType.NONE, r.getFieldInfos().fieldInfo(ID_FIELD).getDocValuesType());
                if (id.equals("2")) {
                    assertTrue(docvalues.advanceExact(i));
                    assertEquals(v, docvalues.binaryValue().utf8ToString());
                } else {
                    assertFalse(docvalues.advanceExact(i));
                }
            }
        }
    });
}