use of org.apache.lucene.facet.taxonomy.FacetLabel in project lucene-solr by apache.
the class TestDirectoryTaxonomyWriter method testRollback.
@Test
public void testRollback() throws Exception {
// Verifies that if rollback is called, DTW is closed.
Directory dir = newDirectory();
DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir);
dtw.addCategory(new FacetLabel("a"));
dtw.rollback();
// should not have succeeded to add a category following rollback.
expectThrows(AlreadyClosedException.class, () -> {
dtw.addCategory(new FacetLabel("a"));
});
dir.close();
}
use of org.apache.lucene.facet.taxonomy.FacetLabel in project lucene-solr by apache.
the class TestDirectoryTaxonomyWriter method testEnsureOpen.
@Test
public void testEnsureOpen() throws Exception {
// verifies that an exception is thrown if DTW was closed
Directory dir = newDirectory();
DirectoryTaxonomyWriter dtw = new DirectoryTaxonomyWriter(dir);
dtw.close();
// should not succeed to add a category following close.
expectThrows(AlreadyClosedException.class, () -> {
dtw.addCategory(new FacetLabel("a"));
});
dir.close();
}
use of org.apache.lucene.facet.taxonomy.FacetLabel in project lucene-solr by apache.
the class TestDirectoryTaxonomyWriter method testReaderFreshness.
@Test
public void testReaderFreshness() throws Exception {
// ensures that the internal index reader is always kept fresh. Previously,
// this simple scenario failed, if the cache just evicted the category that
// is being added.
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, NO_OP_CACHE);
int o1 = taxoWriter.addCategory(new FacetLabel("a"));
int o2 = taxoWriter.addCategory(new FacetLabel("a"));
assertTrue("ordinal for same category that is added twice should be the same !", o1 == o2);
taxoWriter.close();
dir.close();
}
use of org.apache.lucene.facet.taxonomy.FacetLabel in project lucene-solr by apache.
the class TestDirectoryTaxonomyWriter method testConcurrency.
public void testConcurrency() throws Exception {
// add many categories
final int ncats = atLeast(100000);
// affects the categories selection
final int range = ncats * 3;
final AtomicInteger numCats = new AtomicInteger(ncats);
final Directory dir = newDirectory();
final ConcurrentHashMap<String, String> values = new ConcurrentHashMap<>();
final double d = random().nextDouble();
final TaxonomyWriterCache cache;
if (d < 0.7) {
// this is the fastest, yet most memory consuming
cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
} else if (TEST_NIGHTLY && d > 0.98) {
// this is the slowest, but tests the writer concurrency when no caching is done.
// only pick it during NIGHTLY tests, and even then, with very low chances.
cache = NO_OP_CACHE;
} else {
// this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
cache = new LruTaxonomyWriterCache(ncats / 10);
}
if (VERBOSE) {
System.out.println("TEST: use cache=" + cache);
}
final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
Thread[] addThreads = new Thread[atLeast(4)];
for (int z = 0; z < addThreads.length; z++) {
addThreads[z] = new Thread() {
@Override
public void run() {
Random random = random();
while (numCats.decrementAndGet() > 0) {
try {
int value = random.nextInt(range);
FacetLabel cp = new FacetLabel(Integer.toString(value / 1000), Integer.toString(value / 10000), Integer.toString(value / 100000), Integer.toString(value));
int ord = tw.addCategory(cp);
assertTrue("invalid parent for ordinal " + ord + ", category " + cp, tw.getParent(ord) != -1);
String l1 = FacetsConfig.pathToString(cp.components, 1);
String l2 = FacetsConfig.pathToString(cp.components, 2);
String l3 = FacetsConfig.pathToString(cp.components, 3);
String l4 = FacetsConfig.pathToString(cp.components, 4);
values.put(l1, l1);
values.put(l2, l2);
values.put(l3, l3);
values.put(l4, l4);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
};
}
for (Thread t : addThreads) t.start();
for (Thread t : addThreads) t.join();
tw.close();
DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
// +1 for root category
if (values.size() + 1 != dtr.getSize()) {
for (String value : values.keySet()) {
FacetLabel label = new FacetLabel(FacetsConfig.stringToPath(value));
if (dtr.getOrdinal(label) == -1) {
System.out.println("FAIL: path=" + label + " not recognized");
}
}
fail("mismatch number of categories");
}
int[] parents = dtr.getParallelTaxonomyArrays().parents();
for (String cat : values.keySet()) {
FacetLabel cp = new FacetLabel(FacetsConfig.stringToPath(cat));
assertTrue("category not found " + cp, dtr.getOrdinal(cp) > 0);
int level = cp.length;
// for root, parent is always virtual ROOT (ord=0)
int parentOrd = 0;
FacetLabel path = new FacetLabel();
for (int i = 0; i < level; i++) {
path = cp.subpath(i + 1);
int ord = dtr.getOrdinal(path);
assertEquals("invalid parent for cp=" + path, parentOrd, parents[ord]);
// next level should have this parent
parentOrd = ord;
}
}
IOUtils.close(dtr, dir);
}
use of org.apache.lucene.facet.taxonomy.FacetLabel in project lucene-solr by apache.
the class TestDirectoryTaxonomyWriter method testCommitUserData.
@Test
public void testCommitUserData() throws Exception {
// Verifies taxonomy commit data
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
taxoWriter.addCategory(new FacetLabel("a"));
taxoWriter.addCategory(new FacetLabel("b"));
Map<String, String> userCommitData = new HashMap<>();
userCommitData.put("testing", "1 2 3");
taxoWriter.setLiveCommitData(userCommitData.entrySet());
taxoWriter.close();
DirectoryReader r = DirectoryReader.open(dir);
assertEquals("2 categories plus root should have been committed to the underlying directory", 3, r.numDocs());
Map<String, String> readUserCommitData = r.getIndexCommit().getUserData();
assertTrue("wrong value extracted from commit data", "1 2 3".equals(readUserCommitData.get("testing")));
assertNotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData.get(DirectoryTaxonomyWriter.INDEX_EPOCH));
r.close();
// open DirTaxoWriter again and commit, INDEX_EPOCH should still exist
// in the commit data, otherwise DirTaxoReader.refresh() might not detect
// that the taxonomy index has been recreated.
taxoWriter = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE_OR_APPEND, NO_OP_CACHE);
// add a category so that commit will happen
taxoWriter.addCategory(new FacetLabel("c"));
taxoWriter.setLiveCommitData(new HashMap<String, String>() {
{
put("just", "data");
}
}.entrySet());
taxoWriter.commit();
// verify taxoWriter.getCommitData()
Map<String, String> data = new HashMap<>();
Iterable<Map.Entry<String, String>> iter = taxoWriter.getLiveCommitData();
if (iter != null) {
for (Map.Entry<String, String> ent : iter) {
data.put(ent.getKey(), ent.getValue());
}
}
assertNotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in taoxWriter.commitData", data.get(DirectoryTaxonomyWriter.INDEX_EPOCH));
taxoWriter.close();
r = DirectoryReader.open(dir);
readUserCommitData = r.getIndexCommit().getUserData();
assertNotNull(DirectoryTaxonomyWriter.INDEX_EPOCH + " not found in commitData", readUserCommitData.get(DirectoryTaxonomyWriter.INDEX_EPOCH));
r.close();
dir.close();
}
Aggregations