Examples with DirectoryTaxonomyWriter org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter used on opensource projects

Example 56 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestOrdinalMappingLeafReader method buildIndexWithFacets.

private void buildIndexWithFacets(Directory indexDir, Directory taxoDir, boolean asc) throws IOException {
    IndexWriterConfig config = newIndexWriterConfig(null);
    RandomIndexWriter writer = new RandomIndexWriter(random(), indexDir, config);
    DirectoryTaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
    for (int i = 1; i <= NUM_DOCS; i++) {
        Document doc = new Document();
        for (int j = i; j <= NUM_DOCS; j++) {
            int facetValue = asc ? j : NUM_DOCS - j;
            doc.add(new FacetField("tag", Integer.toString(facetValue)));
        }
        // add a facet under default dim config
        doc.add(new FacetField("id", Integer.toString(i)));
        // make sure OrdinalMappingLeafReader ignores non-facet BinaryDocValues fields
        doc.add(new BinaryDocValuesField("bdv", new BytesRef(Integer.toString(i))));
        doc.add(new BinaryDocValuesField("cbdv", new BytesRef(Integer.toString(i * 2))));
        writer.addDocument(facetConfig.build(taxonomyWriter, doc));
    }
    taxonomyWriter.commit();
    taxonomyWriter.close();
    writer.commit();
    writer.close();
}

Example 57 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestOrdinalMappingLeafReader method testTaxonomyMergeUtils.

@Test
public void testTaxonomyMergeUtils() throws Exception {
    Directory srcIndexDir = newDirectory();
    Directory srcTaxoDir = newDirectory();
    buildIndexWithFacets(srcIndexDir, srcTaxoDir, true);
    Directory targetIndexDir = newDirectory();
    Directory targetTaxoDir = newDirectory();
    buildIndexWithFacets(targetIndexDir, targetTaxoDir, false);
    IndexWriter destIndexWriter = new IndexWriter(targetIndexDir, newIndexWriterConfig(null));
    DirectoryTaxonomyWriter destTaxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir);
    try {
        TaxonomyMergeUtils.merge(srcIndexDir, srcTaxoDir, new MemoryOrdinalMap(), destIndexWriter, destTaxoWriter, facetConfig);
    } finally {
        IOUtils.close(destIndexWriter, destTaxoWriter);
    }
    verifyResults(targetIndexDir, targetTaxoDir);
    IOUtils.close(targetIndexDir, targetTaxoDir, srcIndexDir, srcTaxoDir);
}

Example 58 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestSearcherTaxonomyManager method testReplaceTaxonomyNRT.

public void testReplaceTaxonomyNRT() throws Exception {
    Directory dir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    Directory taxoDir2 = newDirectory();
    DirectoryTaxonomyWriter tw2 = new DirectoryTaxonomyWriter(taxoDir2);
    tw2.close();
    SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(w, true, null, tw);
    w.addDocument(new Document());
    tw.replaceTaxonomy(taxoDir2);
    taxoDir2.close();
    expectThrows(IllegalStateException.class, () -> {
        mgr.maybeRefresh();
    });
    w.close();
    IOUtils.close(mgr, tw, taxoDir, dir);
}

Example 59 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestSearcherTaxonomyManager method testNRT.

public void testNRT() throws Exception {
    Directory dir = newDirectory();
    Directory taxoDir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    // Don't allow tiny maxBufferedDocs; it can make this
    // test too slow:
    iwc.setMaxBufferedDocs(Math.max(500, iwc.getMaxBufferedDocs()));
    // MockRandom/AlcololicMergePolicy are too slow:
    TieredMergePolicy tmp = new TieredMergePolicy();
    tmp.setFloorSegmentMB(.001);
    iwc.setMergePolicy(tmp);
    final IndexWriter w = new IndexWriter(dir, iwc);
    final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    final FacetsConfig config = new FacetsConfig();
    config.setMultiValued("field", true);
    final AtomicBoolean stop = new AtomicBoolean();
    // How many unique facets to index before stopping:
    final int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
    Thread indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
    final SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(w, true, null, tw);
    Thread reopener = new Thread() {

        @Override
        public void run() {
            while (!stop.get()) {
                try {
                    // Sleep for up to 20 msec:
                    Thread.sleep(random().nextInt(20));
                    if (VERBOSE) {
                        System.out.println("TEST: reopen");
                    }
                    mgr.maybeRefresh();
                    if (VERBOSE) {
                        System.out.println("TEST: reopen done");
                    }
                } catch (Exception ioe) {
                    throw new RuntimeException(ioe);
                }
            }
        }
    };
    reopener.setName("reopener");
    reopener.start();
    indexer.setName("indexer");
    indexer.start();
    try {
        while (!stop.get()) {
            SearcherAndTaxonomy pair = mgr.acquire();
            try {
                //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
                FacetsCollector sfc = new FacetsCollector();
                pair.searcher.search(new MatchAllDocsQuery(), sfc);
                Facets facets = getTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
                FacetResult result = facets.getTopChildren(10, "field");
                if (pair.searcher.getIndexReader().numDocs() > 0) {
                    //System.out.println(pair.taxonomyReader.getSize());
                    assertTrue(result.childCount > 0);
                    assertTrue(result.labelValues.length > 0);
                }
            //if (VERBOSE) {
            //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
            //}
            } finally {
                mgr.release(pair);
            }
        }
    } finally {
        indexer.join();
        reopener.join();
    }
    if (VERBOSE) {
        System.out.println("TEST: now stop");
    }
    w.close();
    IOUtils.close(mgr, tw, taxoDir, dir);
}

Example 60 with DirectoryTaxonomyWriter

use of org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter in project lucene-solr by apache.

the class TestTaxonomyFacetCounts method testLabelWithDelimiter.

public void testLabelWithDelimiter() throws Exception {
    Directory dir = newDirectory();
    Directory taxoDir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
    FacetsConfig config = new FacetsConfig();
    config.setMultiValued("dim", true);
    Document doc = new Document();
    doc.add(newTextField("field", "text", Field.Store.NO));
    doc.add(new FacetField("dim", "testone"));
    doc.add(new FacetField("dim", "testtwo"));
    writer.addDocument(config.build(taxoWriter, doc));
    // NRT open
    IndexSearcher searcher = newSearcher(writer.getReader());
    // NRT open
    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
    Facets facets = getAllFacets(FacetsConfig.DEFAULT_INDEX_FIELD_NAME, searcher, taxoReader, config);
    assertEquals(1, facets.getSpecificValue("dim", "testone"));
    assertEquals(1, facets.getSpecificValue("dim", "testtwo"));
    FacetResult result = facets.getTopChildren(10, "dim");
    assertEquals("dim=dim path=[] value=-1 childCount=2\n  testone (1)\n  testtwo (1)\n", result.toString());
    writer.close();
    IOUtils.close(taxoWriter, searcher.getIndexReader(), taxoReader, dir, taxoDir);
}