Examples with DirectoryTaxonomyWriter org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter used on opensource projects

Example 6 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestDrillSideways method testMultipleRequestsPerDim.

public void testMultipleRequestsPerDim() throws Exception {
    Directory dir = newDirectory();
    Directory taxoDir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    // Writes facet ords to a separate directory from the
    // main index:
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
    FacetsConfig config = new FacetsConfig();
    config.setHierarchical("dim", true);
    Document doc = new Document();
    doc.add(new FacetField("dim", "a", "x"));
    writer.addDocument(config.build(taxoWriter, doc));
    doc = new Document();
    doc.add(new FacetField("dim", "a", "y"));
    writer.addDocument(config.build(taxoWriter, doc));
    doc = new Document();
    doc.add(new FacetField("dim", "a", "z"));
    writer.addDocument(config.build(taxoWriter, doc));
    doc = new Document();
    doc.add(new FacetField("dim", "b"));
    writer.addDocument(config.build(taxoWriter, doc));
    doc = new Document();
    doc.add(new FacetField("dim", "c"));
    writer.addDocument(config.build(taxoWriter, doc));
    doc = new Document();
    doc.add(new FacetField("dim", "d"));
    writer.addDocument(config.build(taxoWriter, doc));
    // NRT open
    IndexSearcher searcher = newSearcher(writer.getReader());
    //System.out.println("searcher=" + searcher);
    // NRT open
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
    DrillDownQuery ddq = new DrillDownQuery(config);
    ddq.add("dim", "a");
    DrillSidewaysResult r = getNewDrillSideways(searcher, config, taxoReader).search(null, ddq, 10);
    assertEquals(3, r.hits.totalHits);
    assertEquals("dim=dim path=[] value=6 childCount=4\n  a (3)\n  b (1)\n  c (1)\n  d (1)\n", r.facets.getTopChildren(10, "dim").toString());
    assertEquals("dim=dim path=[a] value=3 childCount=3\n  x (1)\n  y (1)\n  z (1)\n", r.facets.getTopChildren(10, "dim", "a").toString());
    writer.close();
    IOUtils.close(searcher.getIndexReader(), taxoReader, taxoWriter, dir, taxoDir);
}

Example 7 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestCachedOrdinalsReader method testWithThreads.

@Test
public void testWithThreads() throws Exception {
    // LUCENE-5303: OrdinalsCache used the ThreadLocal BinaryDV instead of reader.getCoreCacheKey().
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter writer = new IndexWriter(indexDir, conf);
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
    FacetsConfig config = new FacetsConfig();
    Document doc = new Document();
    doc.add(new FacetField("A", "1"));
    writer.addDocument(config.build(taxoWriter, doc));
    doc = new Document();
    doc.add(new FacetField("A", "2"));
    writer.addDocument(config.build(taxoWriter, doc));
    final DirectoryReader reader = DirectoryReader.open(writer);
    final CachedOrdinalsReader ordsReader = new CachedOrdinalsReader(new DocValuesOrdinalsReader(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
    Thread[] threads = new Thread[3];
    for (int i = 0; i < threads.length; i++) {
        threads[i] = new Thread("CachedOrdsThread-" + i) {

            @Override
            public void run() {
                for (LeafReaderContext context : reader.leaves()) {
                    try {
                        ordsReader.getReader(context);
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                }
            }
        };
    }
    long ramBytesUsed = 0;
    for (Thread t : threads) {
        t.start();
        t.join();
        if (ramBytesUsed == 0) {
            ramBytesUsed = ordsReader.ramBytesUsed();
        } else {
            assertEquals(ramBytesUsed, ordsReader.ramBytesUsed());
        }
    }
    writer.close();
    IOUtils.close(taxoWriter, reader, indexDir, taxoDir);
}

Example 8 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestSearcherTaxonomyManager method testDirectory.

public void testDirectory() throws Exception {
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    final IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
    final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    // first empty commit
    w.commit();
    tw.commit();
    final SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
    final FacetsConfig config = new FacetsConfig();
    config.setMultiValued("field", true);
    final AtomicBoolean stop = new AtomicBoolean();
    // How many unique facets to index before stopping:
    final int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
    Thread indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop);
    indexer.start();
    try {
        while (!stop.get()) {
            SearcherAndTaxonomy pair = mgr.acquire();
            try {
                //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
                FacetsCollector sfc = new FacetsCollector();
                pair.searcher.search(new MatchAllDocsQuery(), sfc);
                Facets facets = getTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
                FacetResult result = facets.getTopChildren(10, "field");
                if (pair.searcher.getIndexReader().numDocs() > 0) {
                    //System.out.println(pair.taxonomyReader.getSize());
                    assertTrue(result.childCount > 0);
                    assertTrue(result.labelValues.length > 0);
                }
            //if (VERBOSE) {
            //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
            //}
            } finally {
                mgr.release(pair);
            }
        }
    } finally {
        indexer.join();
    }
    if (VERBOSE) {
        System.out.println("TEST: now stop");
    }
    w.close();
    IOUtils.close(mgr, tw, taxoDir, indexDir);
}

Example 9 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestSearcherTaxonomyManager method testReplaceTaxonomyDirectory.

public void testReplaceTaxonomyDirectory() throws Exception {
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    w.commit();
    tw.commit();
    Directory taxoDir2 = newDirectory();
    DirectoryTaxonomyWriter tw2 = new DirectoryTaxonomyWriter(taxoDir2);
    tw2.addCategory(new FacetLabel("a", "b"));
    tw2.close();
    SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
    SearcherAndTaxonomy pair = mgr.acquire();
    try {
        assertEquals(1, pair.taxonomyReader.getSize());
    } finally {
        mgr.release(pair);
    }
    w.addDocument(new Document());
    tw.replaceTaxonomy(taxoDir2);
    taxoDir2.close();
    w.commit();
    tw.commit();
    mgr.maybeRefresh();
    pair = mgr.acquire();
    try {
        assertEquals(3, pair.taxonomyReader.getSize());
    } finally {
        mgr.release(pair);
    }
    w.close();
    IOUtils.close(mgr, tw, taxoDir, indexDir);
}

Example 10 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestMultipleIndexFields method testSomeSameSomeDifferent.

@Test
public void testSomeSameSomeDifferent() throws Exception {
    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();
    // create and open an index writer
    RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
    // create and open a taxonomy writer
    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
    FacetsConfig config = getConfig();
    config.setIndexFieldName("Band", "$music");
    config.setIndexFieldName("Composer", "$music");
    config.setIndexFieldName("Author", "$literature");
    seedIndex(tw, iw, config);
    IndexReader ir = iw.getReader();
    tw.commit();
    // prepare index reader and taxonomy.
    TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
    // prepare searcher to search against
    IndexSearcher searcher = newSearcher(ir);
    FacetsCollector sfc = performSearch(tr, ir, searcher);
    Map<String, Facets> facetsMap = new HashMap<>();
    Facets facets2 = getTaxonomyFacetCounts(tr, config, sfc, "$music");
    facetsMap.put("Band", facets2);
    facetsMap.put("Composer", facets2);
    facetsMap.put("Author", getTaxonomyFacetCounts(tr, config, sfc, "$literature"));
    Facets facets = new MultiFacets(facetsMap, getTaxonomyFacetCounts(tr, config, sfc));
    // Obtain facets results and hand-test them
    assertCorrectResults(facets);
    assertOrdinalsExist("$music", ir);
    assertOrdinalsExist("$literature", ir);
    iw.close();
    IOUtils.close(tr, ir, iw, tw, indexDir, taxoDir);
}