Examples with DirectoryTaxonomyWriter org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter used on opensource projects

Example 71 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestTaxonomyCombined method testWriter.

/**  Basic tests for TaxonomyWriter. Basically, we test that
    IndexWriter.addCategory works, i.e. returns the expected ordinals
    (this is tested by calling the fillTaxonomy() method above).
    We do not test here that after writing the index can be read -
    this will be done in more tests below.
   */
@Test
public void testWriter() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    fillTaxonomy(tw);
    // Also check TaxonomyWriter.getSize() - see that the taxonomy's size
    // is what we expect it to be.
    assertEquals(expectedCategories.length, tw.getSize());
    tw.close();
    indexDir.close();
}

Example 72 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestTaxonomyCombined method assertConsistentYoungestChild.

private void assertConsistentYoungestChild(final FacetLabel abPath, final int abOrd, final int abYoungChildBase1, final int abYoungChildBase2, final int retry, int numCategories) throws Exception {
    // no slowness for initialization
    SlowRAMDirectory indexDir = new SlowRAMDirectory(-1, null);
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.addCategory(new FacetLabel("a", "0"));
    tw.addCategory(abPath);
    tw.commit();
    final DirectoryTaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    for (int i = 0; i < numCategories; i++) {
        final FacetLabel cp = new FacetLabel("a", "b", Integer.toString(i));
        tw.addCategory(cp);
        assertEquals("Ordinal of " + cp + " must be invalid until Taxonomy Reader was refreshed", TaxonomyReader.INVALID_ORDINAL, tr.getOrdinal(cp));
    }
    tw.close();
    final AtomicBoolean stop = new AtomicBoolean(false);
    final Throwable[] error = new Throwable[] { null };
    final int[] retrieval = { 0 };
    Thread thread = new Thread("Child Arrays Verifier") {

        @Override
        public void run() {
            setPriority(1 + getPriority());
            try {
                while (!stop.get()) {
                    int lastOrd = tr.getParallelTaxonomyArrays().parents().length - 1;
                    assertNotNull("path of last-ord " + lastOrd + " is not found!", tr.getPath(lastOrd));
                    assertChildrenArrays(tr.getParallelTaxonomyArrays(), retry, retrieval[0]++);
                    // don't starve refresh()'s CPU, which sleeps every 50 bytes for 1 ms
                    sleep(10);
                }
            } catch (Throwable e) {
                error[0] = e;
                stop.set(true);
            }
        }

        private void assertChildrenArrays(ParallelTaxonomyArrays ca, int retry, int retrieval) {
            final int abYoungChild = ca.children()[abOrd];
            assertTrue("Retry " + retry + ": retrieval: " + retrieval + ": wrong youngest child for category " + abPath + " (ord=" + abOrd + ") - must be either " + abYoungChildBase1 + " or " + abYoungChildBase2 + " but was: " + abYoungChild, abYoungChildBase1 == abYoungChild || abYoungChildBase2 == ca.children()[abOrd]);
        }
    };
    thread.start();
    // some delay for refresh
    indexDir.setSleepMillis(1);
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(tr);
    if (newTaxoReader != null) {
        newTaxoReader.close();
    }
    stop.set(true);
    thread.join();
    assertNull("Unexpcted exception at retry " + retry + " retrieval " + retrieval[0] + ": \n" + stackTraceStr(error[0]), error[0]);
    tr.close();
}

Example 73 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestTaxonomyCombined method testSeparateReaderAndWriter.

/**  Test that if separate reader and writer objects are opened, new
    categories written into the writer are available to a reader only
    after a commit().
    Note that this test obviously doesn't cover all the different
    concurrency scenarios, all different methods, and so on. We may
    want to write more tests of this sort.

    This test simulates what would happen when there are two separate
    processes, one doing indexing, and the other searching, and each opens
    its own object (with obviously no connection between the objects) using
    the same disk files. Note, though, that this test does not test what
    happens when the two processes do their actual work at exactly the same
    time.
    It also doesn't test multi-threading.
   */
@Test
public void testSeparateReaderAndWriter() throws Exception {
    Directory indexDir = newDirectory();
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(indexDir);
    tw.commit();
    TaxonomyReader tr = new DirectoryTaxonomyReader(indexDir);
    // the empty taxonomy has size 1 (the root)
    assertEquals(1, tr.getSize());
    tw.addCategory(new FacetLabel("Author"));
    // still root only...
    assertEquals(1, tr.getSize());
    // this is not enough, because tw.commit() hasn't been done yet
    assertNull(TaxonomyReader.openIfChanged(tr));
    // still root only...
    assertEquals(1, tr.getSize());
    tw.commit();
    // still root only...
    assertEquals(1, tr.getSize());
    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(tr);
    assertNotNull(newTaxoReader);
    tr.close();
    tr = newTaxoReader;
    int author = 1;
    try {
        assertEquals(TaxonomyReader.ROOT_ORDINAL, tr.getParallelTaxonomyArrays().parents()[author]);
    // ok
    } catch (ArrayIndexOutOfBoundsException e) {
        fail("After category addition, commit() and refresh(), getParent for " + author + " should NOT throw exception");
    }
    // finally, see there are two categories
    assertEquals(2, tr.getSize());
    // now, add another category, and verify that after commit and refresh
    // the parent of this category is correct (this requires the reader
    // to correctly update its prefetched parent vector), and that the
    // old information also wasn't ruined:
    tw.addCategory(new FacetLabel("Author", "Richard Dawkins"));
    int dawkins = 2;
    tw.commit();
    newTaxoReader = TaxonomyReader.openIfChanged(tr);
    assertNotNull(newTaxoReader);
    tr.close();
    tr = newTaxoReader;
    int[] parents = tr.getParallelTaxonomyArrays().parents();
    assertEquals(author, parents[dawkins]);
    assertEquals(TaxonomyReader.ROOT_ORDINAL, parents[author]);
    assertEquals(TaxonomyReader.INVALID_ORDINAL, parents[TaxonomyReader.ROOT_ORDINAL]);
    assertEquals(3, tr.getSize());
    tw.close();
    tr.close();
    indexDir.close();
}

Example 74 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestTaxonomyFacetCounts method testGetFacetResultsTwice.

public void testGetFacetResultsTwice() throws Exception {
    // LUCENE-4893: counts were multiplied as many times as getFacetResults was called.
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
    FacetsConfig config = new FacetsConfig();
    Document doc = new Document();
    doc.add(new FacetField("a", "1"));
    doc.add(new FacetField("b", "1"));
    iw.addDocument(config.build(taxoWriter, doc));
    DirectoryReader r = DirectoryReader.open(iw);
    DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
    Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, newSearcher(r), taxoReader, config);
    List<FacetResult> res1 = facets.getAllDims(10);
    List<FacetResult> res2 = facets.getAllDims(10);
    assertEquals("calling getFacetResults twice should return the .equals()=true result", res1, res2);
    iw.close();
    IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}

Example 75 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestTaxonomyFacetCounts method testCountRoot.

public void testCountRoot() throws Exception {
    // LUCENE-4882: FacetsAccumulator threw NPE if a FacetRequest was defined on CP.EMPTY
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
    FacetsConfig config = new FacetsConfig();
    for (int i = atLeast(30); i > 0; --i) {
        Document doc = new Document();
        doc.add(new FacetField("a", "1"));
        doc.add(new FacetField("b", "1"));
        iw.addDocument(config.build(taxoWriter, doc));
    }
    DirectoryReader r = DirectoryReader.open(iw);
    DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
    Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, newSearcher(r), taxoReader, config);
    for (FacetResult result : facets.getAllDims(10)) {
        assertEquals(r.numDocs(), result.value.intValue());
    }
    iw.close();
    IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}