use of org.apache.lucene.spatial.prefix.tree.Cell in project lucene-solr by apache.
the class NumberRangeFacetsTest method test.
@Repeat(iterations = 20)
@Test
public void test() throws IOException {
//generate test data
List<Shape> indexedShapes = new ArrayList<>();
final int numIndexedShapes = random().nextInt(15);
for (int i = 0; i < numIndexedShapes; i++) {
indexedShapes.add(randomShape());
}
//Main index loop:
for (int i = 0; i < indexedShapes.size(); i++) {
Shape shape = indexedShapes.get(i);
adoc("" + i, shape);
if (random().nextInt(10) == 0)
//intermediate commit, produces extra segments
commit();
}
//delete some documents randomly
for (int id = 0; id < indexedShapes.size(); id++) {
if (random().nextInt(10) == 0) {
deleteDoc("" + id);
indexedShapes.set(id, null);
}
}
commit();
//Main query loop:
for (int queryIdx = 0; queryIdx < 10; queryIdx++) {
preQueryHavoc();
// We need to have a facet range window to do the facets between (a start time & end time). We randomly
// pick a date, decide the level we want to facet on, and then pick a right end time that is up to 2 thousand
// values later.
int calFieldFacet = randomCalWindowField - 1;
if (calFieldFacet > 1 && rarely()) {
calFieldFacet--;
}
final Calendar leftCal = randomCalendar();
leftCal.add(calFieldFacet, -1 * randomInt(1000));
Calendar rightCal = (Calendar) leftCal.clone();
rightCal.add(calFieldFacet, randomInt(2000));
// Pick facet detail level based on cal field.
int detailLevel = tree.getTreeLevelForCalendarField(calFieldFacet);
if (detailLevel < 0) {
//no exact match
detailLevel = -1 * detailLevel;
}
//Randomly pick a filter/acceptDocs
Bits topAcceptDocs = null;
List<Integer> acceptFieldIds = new ArrayList<>();
if (usually()) {
// replace the list.
for (int i = 0; i < indexedShapes.size(); i++) {
if (indexedShapes.get(i) == null) {
// we deleted this one
continue;
}
acceptFieldIds.add(i);
}
Collections.shuffle(acceptFieldIds, random());
acceptFieldIds = acceptFieldIds.subList(0, randomInt(acceptFieldIds.size()));
if (!acceptFieldIds.isEmpty()) {
List<BytesRef> terms = new ArrayList<>();
for (Integer acceptDocId : acceptFieldIds) {
terms.add(new BytesRef(acceptDocId.toString()));
}
topAcceptDocs = searchForDocBits(new TermInSetQuery("id", terms));
}
}
//Lets do it!
NumberRangePrefixTree.NRShape facetRange = tree.toRangeShape(tree.toShape(leftCal), tree.toShape(rightCal));
Facets facets = ((NumberRangePrefixTreeStrategy) strategy).calcFacets(indexSearcher.getTopReaderContext(), topAcceptDocs, facetRange, detailLevel);
//System.out.println("Q: " + queryIdx + " " + facets);
//Verify results. We do it by looping over indexed shapes and reducing the facet counts.
Shape facetShapeRounded = facetRange.roundToLevel(detailLevel);
for (int indexedShapeId = 0; indexedShapeId < indexedShapes.size(); indexedShapeId++) {
if (topAcceptDocs != null && !acceptFieldIds.contains(indexedShapeId)) {
// this doc was filtered out via acceptDocs
continue;
}
Shape indexedShape = indexedShapes.get(indexedShapeId);
if (indexedShape == null) {
//was deleted
continue;
}
Shape indexedShapeRounded = ((NumberRangePrefixTree.NRShape) indexedShape).roundToLevel(detailLevel);
if (!indexedShapeRounded.relate(facetShapeRounded).intersects()) {
// no intersection at all
continue;
}
// walk the cells
final CellIterator cellIterator = tree.getTreeCellIterator(indexedShape, detailLevel);
while (cellIterator.hasNext()) {
Cell cell = cellIterator.next();
if (!cell.getShape().relate(facetShapeRounded).intersects()) {
//no intersection; prune
cellIterator.remove();
continue;
}
assert cell.getLevel() <= detailLevel;
if (cell.getLevel() == detailLevel) {
//count it
UnitNRShape shape = (UnitNRShape) cell.getShape();
//get parent
final UnitNRShape parentShape = shape.getShapeAtLevel(detailLevel - 1);
final Facets.FacetParentVal facetParentVal = facets.parents.get(parentShape);
assertNotNull(facetParentVal);
int index = shape.getValAtLevel(shape.getLevel());
assertNotNull(facetParentVal.childCounts);
assert facetParentVal.childCounts[index] > 0;
facetParentVal.childCounts[index]--;
} else if (cell.isLeaf()) {
//count it, and remove/prune.
if (cell.getLevel() < detailLevel - 1) {
assert facets.topLeaves > 0;
facets.topLeaves--;
} else {
UnitNRShape shape = (UnitNRShape) cell.getShape();
//get parent
final UnitNRShape parentShape = shape.getShapeAtLevel(detailLevel - 1);
final Facets.FacetParentVal facetParentVal = facets.parents.get(parentShape);
assertNotNull(facetParentVal);
assert facetParentVal.parentLeaves > 0;
facetParentVal.parentLeaves--;
}
cellIterator.remove();
}
}
}
// At this point; all counts should be down to zero.
assertTrue(facets.topLeaves == 0);
for (Facets.FacetParentVal facetParentVal : facets.parents.values()) {
assertTrue(facetParentVal.parentLeaves == 0);
if (facetParentVal.childCounts != null) {
for (int childCount : facetParentVal.childCounts) {
assertTrue(childCount == 0);
}
}
}
}
}
use of org.apache.lucene.spatial.prefix.tree.Cell in project lucene-solr by apache.
the class RandomSpatialOpFuzzyPrefixTreeTest method gridSnap.
// private Rectangle inset(Rectangle r) {
// //typically inset by 1 (whole numbers are easy to read)
// double d = Math.min(1.0, grid.getDistanceForLevel(grid.getMaxLevels()) / 4);
// return ctx.makeRectangle(r.getMinX() + d, r.getMaxX() - d, r.getMinY() + d, r.getMaxY() - d);
// }
protected Shape gridSnap(Shape snapMe) {
if (snapMe == null)
return null;
if (snapMe instanceof ShapePair) {
ShapePair me = (ShapePair) snapMe;
return new ShapePair(gridSnap(me.shape1), gridSnap(me.shape2), me.biasContainsThenWithin);
}
if (snapMe instanceof Point) {
snapMe = snapMe.getBoundingBox();
}
//The next 4 lines mimic PrefixTreeStrategy.createIndexableFields()
double distErrPct = ((PrefixTreeStrategy) strategy).getDistErrPct();
double distErr = SpatialArgs.calcDistanceFromErrPct(snapMe, distErrPct, ctx);
int detailLevel = grid.getLevelForDistance(distErr);
CellIterator cells = grid.getTreeCellIterator(snapMe, detailLevel);
//calc bounding box of cells.
List<Shape> cellShapes = new ArrayList<>(1024);
while (cells.hasNext()) {
Cell cell = cells.next();
if (!cell.isLeaf())
continue;
cellShapes.add(cell.getShape());
}
return new ShapeCollection<>(cellShapes, ctx).getBoundingBox();
}
use of org.apache.lucene.spatial.prefix.tree.Cell in project lucene-solr by apache.
the class TermQueryPrefixTreeStrategy method makeQuery.
@Override
public Query makeQuery(SpatialArgs args) {
final SpatialOperation op = args.getOperation();
if (op != SpatialOperation.Intersects)
throw new UnsupportedSpatialOperation(op);
Shape shape = args.getShape();
int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
//--get a List of BytesRef for each term we want (no parents, no leaf bytes))
final int GUESS_NUM_TERMS;
if (shape instanceof Point)
//perfect guess
GUESS_NUM_TERMS = detailLevel;
else
//should this be a method on SpatialPrefixTree?
GUESS_NUM_TERMS = 4096;
//shared byte array for all terms
BytesRefBuilder masterBytes = new BytesRefBuilder();
List<BytesRef> terms = new ArrayList<>(GUESS_NUM_TERMS);
CellIterator cells = grid.getTreeCellIterator(shape, detailLevel);
while (cells.hasNext()) {
Cell cell = cells.next();
if (!cell.isLeaf())
continue;
//null because we want a new BytesRef
BytesRef term = cell.getTokenBytesNoLeaf(null);
//We copy out the bytes because it may be re-used across the iteration. This also gives us the opportunity
// to use one contiguous block of memory for the bytes of all terms we need.
masterBytes.grow(masterBytes.length() + term.length);
masterBytes.append(term);
//don't need; will reset later
term.bytes = null;
term.offset = masterBytes.length() - term.length;
terms.add(term);
}
//doing this now because if we did earlier, it's possible the bytes needed to grow()
for (BytesRef byteRef : terms) {
byteRef.bytes = masterBytes.bytes();
}
//TODO an automatonQuery might be faster?
return new TermInSetQuery(getFieldName(), terms);
}
use of org.apache.lucene.spatial.prefix.tree.Cell in project lucene-solr by apache.
the class WithinPrefixTreeQuery method getDocIdSet.
@Override
protected DocIdSet getDocIdSet(LeafReaderContext context) throws IOException {
return new VisitorTemplate(context) {
private FixedBitSet inside;
private FixedBitSet outside;
@Override
protected void start() {
inside = new FixedBitSet(maxDoc);
outside = new FixedBitSet(maxDoc);
}
@Override
protected DocIdSet finish() {
inside.andNot(outside);
return new BitDocIdSet(inside);
}
@Override
protected CellIterator findSubCellsToVisit(Cell cell) {
//use buffered query shape instead of orig. Works with null too.
return cell.getNextLevelCells(bufferedQueryShape);
}
@Override
protected boolean visitPrefix(Cell cell) throws IOException {
//cell.relate is based on the bufferedQueryShape; we need to examine what
// the relation is against the queryShape
SpatialRelation visitRelation = cell.getShape().relate(queryShape);
if (cell.getLevel() == detailLevel) {
collectDocs(visitRelation.intersects() ? inside : outside);
return false;
} else if (visitRelation == SpatialRelation.WITHIN) {
collectDocs(inside);
return false;
} else if (visitRelation == SpatialRelation.DISJOINT) {
collectDocs(outside);
return false;
}
return true;
}
@Override
protected void visitLeaf(Cell cell) throws IOException {
if (allCellsIntersectQuery(cell))
collectDocs(inside);
else
collectDocs(outside);
}
/** Returns true if the provided cell, and all its sub-cells down to
* detailLevel all intersect the queryShape.
*/
private boolean allCellsIntersectQuery(Cell cell) {
SpatialRelation relate = cell.getShape().relate(queryShape);
if (cell.getLevel() == detailLevel)
return relate.intersects();
if (relate == SpatialRelation.WITHIN)
return true;
if (relate == SpatialRelation.DISJOINT)
return false;
// Note: Generating all these cells just to determine intersection is not ideal.
// The real solution is LUCENE-4869.
CellIterator subCells = cell.getNextLevelCells(null);
while (subCells.hasNext()) {
Cell subCell = subCells.next();
if (//recursion
!allCellsIntersectQuery(subCell))
return false;
}
return true;
}
@Override
protected void visitScanned(Cell cell) throws IOException {
//collects as we want, even if not a leaf
visitLeaf(cell);
// if (cell.isLeaf()) {
// visitLeaf(cell);
// } else {
// visitPrefix(cell);
// }
}
}.getDocIdSet();
}
use of org.apache.lucene.spatial.prefix.tree.Cell in project lucene-solr by apache.
the class NumberRangePrefixTreeStrategy method calcFacets.
/**
* Calculates facets (aggregated counts) given a range shape (start-end span) and a level, which specifies the detail.
* To get the level of an existing shape, say a Calendar, call
* {@link org.apache.lucene.spatial.prefix.tree.NumberRangePrefixTree#toUnitShape(Object)} then call
* {@link org.apache.lucene.spatial.prefix.tree.NumberRangePrefixTree.UnitNRShape#getLevel()}.
* Facet computation is implemented by navigating the underlying indexed terms efficiently.
*/
public Facets calcFacets(IndexReaderContext context, Bits topAcceptDocs, Shape facetRange, final int level) throws IOException {
final Facets facets = new Facets(level);
PrefixTreeFacetCounter.compute(this, context, topAcceptDocs, facetRange, level, new PrefixTreeFacetCounter.FacetVisitor() {
Facets.FacetParentVal parentFacet;
UnitNRShape parentShape;
@Override
public void visit(Cell cell, int count) {
if (cell.getLevel() < level - 1) {
//some ancestor of parent facet level, direct or distant
//reset
parentFacet = null;
//reset
parentShape = null;
facets.topLeaves += count;
} else if (cell.getLevel() == level - 1) {
//parent
//set up FacetParentVal
setupParent((UnitNRShape) cell.getShape());
parentFacet.parentLeaves += count;
} else {
//at facet level
UnitNRShape unitShape = (UnitNRShape) cell.getShape();
UnitNRShape unitShapeParent = unitShape.getShapeAtLevel(unitShape.getLevel() - 1);
if (parentFacet == null || !parentShape.equals(unitShapeParent)) {
setupParent(unitShapeParent);
}
//lazy init childCounts
if (parentFacet.childCounts == null) {
parentFacet.childCounts = new int[parentFacet.childCountsLen];
}
parentFacet.childCounts[unitShape.getValAtLevel(cell.getLevel())] += count;
}
}
private void setupParent(UnitNRShape unitShape) {
parentShape = unitShape.clone();
//Look for existing parentFacet (from previous segment), or create anew if needed
parentFacet = facets.parents.get(parentShape);
if (parentFacet == null) {
//didn't find one; make a new one
parentFacet = new Facets.FacetParentVal();
parentFacet.childCountsLen = getGrid().getNumSubCells(parentShape);
facets.parents.put(parentShape, parentFacet);
}
}
});
return facets;
}
Aggregations