use of org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField in project lucene-solr by apache.
the class FacetsConfig method processSSDVFacetFields.
private void processSSDVFacetFields(Map<String, List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
//System.out.println("process SSDV: " + byField);
for (Map.Entry<String, List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {
String indexFieldName = ent.getKey();
for (SortedSetDocValuesFacetField facetField : ent.getValue()) {
FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
String fullPath = pathToString(cp.components, cp.length);
//System.out.println("add " + fullPath);
// For facet counts:
doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));
// For drill-down:
doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));
doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
}
}
}
use of org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField in project lucene-solr by apache.
the class TestDrillSideways method testRandom.
public void testRandom() throws Exception {
while (aChance == 0.0) {
aChance = random().nextDouble();
}
while (bChance == 0.0) {
bChance = random().nextDouble();
}
while (cChance == 0.0) {
cChance = random().nextDouble();
}
//aChance = .01;
//bChance = 0.5;
//cChance = 1.0;
double sum = aChance + bChance + cChance;
aChance /= sum;
bChance /= sum;
cChance /= sum;
int numDims = TestUtil.nextInt(random(), 2, 5);
//int numDims = 3;
int numDocs = atLeast(3000);
//int numDocs = 20;
if (VERBOSE) {
System.out.println("numDims=" + numDims + " numDocs=" + numDocs + " aChance=" + aChance + " bChance=" + bChance + " cChance=" + cChance);
}
String[][] dimValues = new String[numDims][];
int valueCount = 2;
for (int dim = 0; dim < numDims; dim++) {
Set<String> values = new HashSet<>();
while (values.size() < valueCount) {
String s = TestUtil.randomRealisticUnicodeString(random());
//String s = _TestUtil.randomString(random());
if (s.length() > 0) {
values.add(s);
}
}
dimValues[dim] = values.toArray(new String[values.size()]);
valueCount *= 2;
}
List<Doc> docs = new ArrayList<>();
for (int i = 0; i < numDocs; i++) {
Doc doc = new Doc();
doc.id = "" + i;
doc.contentToken = randomContentToken(false);
doc.dims = new int[numDims];
doc.dims2 = new int[numDims];
for (int dim = 0; dim < numDims; dim++) {
if (random().nextInt(5) == 3) {
// This doc is missing this dim:
doc.dims[dim] = -1;
} else if (dimValues[dim].length <= 4) {
int dimUpto = 0;
doc.dims[dim] = dimValues[dim].length - 1;
while (dimUpto < dimValues[dim].length) {
if (random().nextBoolean()) {
doc.dims[dim] = dimUpto;
break;
}
dimUpto++;
}
} else {
doc.dims[dim] = random().nextInt(dimValues[dim].length);
}
if (random().nextInt(5) == 3) {
// 2nd value:
doc.dims2[dim] = random().nextInt(dimValues[dim].length);
} else {
doc.dims2[dim] = -1;
}
}
docs.add(doc);
}
Directory d = newDirectory();
Directory td = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
iwc.setInfoStream(InfoStream.NO_OUTPUT);
RandomIndexWriter w = new RandomIndexWriter(random(), d, iwc);
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(td, IndexWriterConfig.OpenMode.CREATE);
FacetsConfig config = new FacetsConfig();
for (int i = 0; i < numDims; i++) {
config.setMultiValued("dim" + i, true);
}
boolean doUseDV = random().nextBoolean();
for (Doc rawDoc : docs) {
Document doc = new Document();
doc.add(newStringField("id", rawDoc.id, Field.Store.YES));
doc.add(new SortedDocValuesField("id", new BytesRef(rawDoc.id)));
doc.add(newStringField("content", rawDoc.contentToken, Field.Store.NO));
if (VERBOSE) {
System.out.println(" doc id=" + rawDoc.id + " token=" + rawDoc.contentToken);
}
for (int dim = 0; dim < numDims; dim++) {
int dimValue = rawDoc.dims[dim];
if (dimValue != -1) {
if (doUseDV) {
doc.add(new SortedSetDocValuesFacetField("dim" + dim, dimValues[dim][dimValue]));
} else {
doc.add(new FacetField("dim" + dim, dimValues[dim][dimValue]));
}
doc.add(new StringField("dim" + dim, dimValues[dim][dimValue], Field.Store.YES));
if (VERBOSE) {
System.out.println(" dim" + dim + "=" + new BytesRef(dimValues[dim][dimValue]));
}
}
int dimValue2 = rawDoc.dims2[dim];
if (dimValue2 != -1) {
if (doUseDV) {
doc.add(new SortedSetDocValuesFacetField("dim" + dim, dimValues[dim][dimValue2]));
} else {
doc.add(new FacetField("dim" + dim, dimValues[dim][dimValue2]));
}
doc.add(new StringField("dim" + dim, dimValues[dim][dimValue2], Field.Store.YES));
if (VERBOSE) {
System.out.println(" dim" + dim + "=" + new BytesRef(dimValues[dim][dimValue2]));
}
}
}
w.addDocument(config.build(tw, doc));
}
if (random().nextBoolean()) {
// Randomly delete a few docs:
int numDel = TestUtil.nextInt(random(), 1, (int) (numDocs * 0.05));
if (VERBOSE) {
System.out.println("delete " + numDel);
}
int delCount = 0;
while (delCount < numDel) {
Doc doc = docs.get(random().nextInt(docs.size()));
if (!doc.deleted) {
if (VERBOSE) {
System.out.println(" delete id=" + doc.id);
}
doc.deleted = true;
w.deleteDocuments(new Term("id", doc.id));
delCount++;
}
}
}
if (random().nextBoolean()) {
if (VERBOSE) {
System.out.println("TEST: forceMerge(1)...");
}
w.forceMerge(1);
}
IndexReader r = w.getReader();
final SortedSetDocValuesReaderState sortedSetDVState;
IndexSearcher s = newSearcher(r);
if (doUseDV) {
sortedSetDVState = new DefaultSortedSetDocValuesReaderState(s.getIndexReader());
} else {
sortedSetDVState = null;
}
if (VERBOSE) {
System.out.println("r.numDocs() = " + r.numDocs());
}
// NRT open
TaxonomyReader tr = new DirectoryTaxonomyReader(tw);
int numIters = atLeast(10);
for (int iter = 0; iter < numIters; iter++) {
String contentToken = random().nextInt(30) == 17 ? null : randomContentToken(true);
int numDrillDown = TestUtil.nextInt(random(), 1, Math.min(4, numDims));
if (VERBOSE) {
System.out.println("\nTEST: iter=" + iter + " baseQuery=" + contentToken + " numDrillDown=" + numDrillDown + " useSortedSetDV=" + doUseDV);
}
String[][] drillDowns = new String[numDims][];
int count = 0;
boolean anyMultiValuedDrillDowns = false;
while (count < numDrillDown) {
int dim = random().nextInt(numDims);
if (drillDowns[dim] == null) {
if (random().nextBoolean()) {
// Drill down on one value:
drillDowns[dim] = new String[] { dimValues[dim][random().nextInt(dimValues[dim].length)] };
} else {
int orCount = TestUtil.nextInt(random(), 1, Math.min(5, dimValues[dim].length));
drillDowns[dim] = new String[orCount];
anyMultiValuedDrillDowns |= orCount > 1;
for (int i = 0; i < orCount; i++) {
while (true) {
String value = dimValues[dim][random().nextInt(dimValues[dim].length)];
for (int j = 0; j < i; j++) {
if (value.equals(drillDowns[dim][j])) {
value = null;
break;
}
}
if (value != null) {
drillDowns[dim][i] = value;
break;
}
}
}
}
if (VERBOSE) {
BytesRef[] values = new BytesRef[drillDowns[dim].length];
for (int i = 0; i < values.length; i++) {
values[i] = new BytesRef(drillDowns[dim][i]);
}
System.out.println(" dim" + dim + "=" + Arrays.toString(values));
}
count++;
}
}
Query baseQuery;
if (contentToken == null) {
baseQuery = new MatchAllDocsQuery();
} else {
baseQuery = new TermQuery(new Term("content", contentToken));
}
DrillDownQuery ddq = new DrillDownQuery(config, baseQuery);
for (int dim = 0; dim < numDims; dim++) {
if (drillDowns[dim] != null) {
for (String value : drillDowns[dim]) {
ddq.add("dim" + dim, value);
}
}
}
Query filter;
if (random().nextInt(7) == 6) {
if (VERBOSE) {
System.out.println(" only-even filter");
}
filter = new Query() {
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
return new ConstantScoreWeight(this, boost) {
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
return new ConstantScoreScorer(this, score(), new TwoPhaseIterator(approximation) {
@Override
public boolean matches() throws IOException {
int docID = approximation.docID();
return (Integer.parseInt(context.reader().document(docID).get("id")) & 1) == 0;
}
@Override
public float matchCost() {
return 1000f;
}
});
}
};
}
@Override
public String toString(String field) {
return "drillSidewaysTestFilter";
}
@Override
public boolean equals(Object o) {
return o == this;
}
@Override
public int hashCode() {
return System.identityHashCode(this);
}
};
} else {
filter = null;
}
// Verify docs are always collected in order. If we
// had an AssertingScorer it could catch it when
// Weight.scoresDocsOutOfOrder lies!:
getNewDrillSideways(s, config, tr).search(ddq, new SimpleCollector() {
int lastDocID;
@Override
public void collect(int doc) {
assert doc > lastDocID;
lastDocID = doc;
}
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
lastDocID = -1;
}
@Override
public boolean needsScores() {
return false;
}
});
// subScorers are on the same docID:
if (!anyMultiValuedDrillDowns) {
// Can only do this test when there are no OR'd
// drill-down values, because in that case it's
// easily possible for one of the DD terms to be on
// a future docID:
getNewDrillSidewaysScoreSubdocsAtOnce(s, config, tr).search(ddq, new AssertingSubDocsAtOnceCollector());
}
TestFacetResult expected = slowDrillSidewaysSearch(s, docs, contentToken, drillDowns, dimValues, filter);
Sort sort = new Sort(new SortField("id", SortField.Type.STRING));
DrillSideways ds;
if (doUseDV) {
ds = getNewDrillSideways(s, config, sortedSetDVState);
} else {
ds = getNewDrillSidewaysBuildFacetsResult(s, config, tr);
}
// Retrieve all facets:
DrillSidewaysResult actual = ds.search(ddq, filter, null, numDocs, sort, true, true);
TopDocs hits = s.search(baseQuery, numDocs);
Map<String, Float> scores = new HashMap<>();
for (ScoreDoc sd : hits.scoreDocs) {
scores.put(s.doc(sd.doc).get("id"), sd.score);
}
if (VERBOSE) {
System.out.println(" verify all facets");
}
verifyEquals(dimValues, s, expected, actual, scores, doUseDV);
// Make sure drill down doesn't change score:
Query q = ddq;
if (filter != null) {
q = new BooleanQuery.Builder().add(q, Occur.MUST).add(filter, Occur.FILTER).build();
}
TopDocs ddqHits = s.search(q, numDocs);
assertEquals(expected.hits.size(), ddqHits.totalHits);
for (int i = 0; i < expected.hits.size(); i++) {
// Score should be IDENTICAL:
assertEquals(scores.get(expected.hits.get(i).id), ddqHits.scoreDocs[i].score, 0.0f);
}
}
w.close();
IOUtils.close(r, tr, tw, d, td);
}
use of org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField in project lucene-solr by apache.
the class TestFacetLabel method testEmptyNullComponents.
@Test
public void testEmptyNullComponents() throws Exception {
// LUCENE-4724: CategoryPath should not allow empty or null components
String[][] components_tests = new String[][] { // empty in the beginning
new String[] { "", "test" }, // empty in the end
new String[] { "test", "" }, // empty in the middle
new String[] { "test", "", "foo" }, // null at the beginning
new String[] { null, "test" }, // null in the end
new String[] { "test", null }, // null in the middle
new String[] { "test", null, "foo" } };
// empty or null components should not be allowed.
for (String[] components : components_tests) {
expectThrows(IllegalArgumentException.class, () -> {
new FacetLabel(components);
});
expectThrows(IllegalArgumentException.class, () -> {
new FacetField("dim", components);
});
expectThrows(IllegalArgumentException.class, () -> {
new AssociationFacetField(new BytesRef(), "dim", components);
});
expectThrows(IllegalArgumentException.class, () -> {
new IntAssociationFacetField(17, "dim", components);
});
expectThrows(IllegalArgumentException.class, () -> {
new FloatAssociationFacetField(17.0f, "dim", components);
});
}
expectThrows(IllegalArgumentException.class, () -> {
new FacetField(null, new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new FacetField("", new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new IntAssociationFacetField(17, null, new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new IntAssociationFacetField(17, "", new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new FloatAssociationFacetField(17.0f, null, new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new FloatAssociationFacetField(17.0f, "", new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new AssociationFacetField(new BytesRef(), null, new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new AssociationFacetField(new BytesRef(), "", new String[] { "abc" });
});
expectThrows(IllegalArgumentException.class, () -> {
new SortedSetDocValuesFacetField(null, "abc");
});
expectThrows(IllegalArgumentException.class, () -> {
new SortedSetDocValuesFacetField("", "abc");
});
expectThrows(IllegalArgumentException.class, () -> {
new SortedSetDocValuesFacetField("dim", null);
});
expectThrows(IllegalArgumentException.class, () -> {
new SortedSetDocValuesFacetField("dim", "");
});
}
use of org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField in project lucene-solr by apache.
the class SimpleSortedSetFacetsExample method index.
/** Build the example index. */
private void index() throws IOException {
IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE));
Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("Author", "Bob"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2010"));
indexWriter.addDocument(config.build(doc));
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("Author", "Lisa"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2010"));
indexWriter.addDocument(config.build(doc));
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("Author", "Lisa"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2012"));
indexWriter.addDocument(config.build(doc));
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("Author", "Susan"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "2012"));
indexWriter.addDocument(config.build(doc));
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("Author", "Frank"));
doc.add(new SortedSetDocValuesFacetField("Publish Year", "1999"));
indexWriter.addDocument(config.build(doc));
indexWriter.close();
}
use of org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField in project jackrabbit-oak by apache.
the class LuceneDocumentMaker method addFacetFields.
private boolean addFacetFields(List<Field> fields, PropertyState property, String pname, PropertyDefinition pd) {
String facetFieldName = FieldNames.createFacetFieldName(pname);
getFacetsConfig().setIndexFieldName(pname, facetFieldName);
int tag = property.getType().tag();
int idxDefinedTag = pd.getType();
// Try converting type to the defined type in the index definition
if (tag != idxDefinedTag) {
log.debug("[{}] Facet property defined with type {} differs from property {} with type {} in " + "path {}", getIndexName(), Type.fromTag(idxDefinedTag, false), property.toString(), Type.fromTag(tag, false), path);
tag = idxDefinedTag;
}
boolean fieldAdded = false;
try {
if (tag == Type.STRINGS.tag() && property.isArray()) {
getFacetsConfig().setMultiValued(pname, true);
Iterable<String> values = property.getValue(Type.STRINGS);
for (String value : values) {
if (value != null && value.length() > 0) {
fields.add(new SortedSetDocValuesFacetField(pname, value));
}
}
fieldAdded = true;
} else if (tag == Type.STRING.tag()) {
String value = property.getValue(Type.STRING);
if (value.length() > 0) {
fields.add(new SortedSetDocValuesFacetField(pname, value));
fieldAdded = true;
}
}
} catch (Throwable e) {
log.warn("[{}] Ignoring facet property. Could not convert property {} of type {} to type {} for path {}", getIndexName(), pname, Type.fromTag(property.getType().tag(), false), Type.fromTag(tag, false), path, e);
}
return fieldAdded;
}
Aggregations