use of org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor in project asterixdb by apache.
the class LSMRTreeWithAntiMatterTuplesSearchCursor method open.
@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
cmp = lsmInitialState.getHilbertCmp();
btreeCmp = lsmInitialState.getBTreeCmp();
lsmHarness = lsmInitialState.getLSMHarness();
comparatorFields = lsmInitialState.getComparatorFields();
operationalComponents = lsmInitialState.getOperationalComponents();
rtreeSearchPredicate = (SearchPredicate) searchPred;
searchCallback = lsmInitialState.getSearchOperationCallback();
includeMutableComponent = false;
numMutableComponents = 0;
int numImmutableComponents = 0;
for (ILSMComponent component : operationalComponents) {
if (component.getType() == LSMComponentType.MEMORY) {
includeMutableComponent = true;
numMutableComponents++;
} else {
numImmutableComponents++;
}
}
if (includeMutableComponent) {
btreeRangePredicate = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
}
mutableRTreeCursors = new RTreeSearchCursor[numMutableComponents];
mutableRTreeAccessors = new ITreeIndexAccessor[numMutableComponents];
btreeCursors = new BTreeRangeSearchCursor[numMutableComponents];
btreeAccessors = new ITreeIndexAccessor[numMutableComponents];
for (int i = 0; i < numMutableComponents; i++) {
ILSMComponent component = operationalComponents.get(i);
RTree rtree = ((LSMRTreeMemoryComponent) component).getRTree();
BTree btree = ((LSMRTreeMemoryComponent) component).getBTree();
mutableRTreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame());
btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame(), false);
btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
mutableRTreeAccessors[i] = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
}
rangeCursors = new RTreeSearchCursor[numImmutableComponents];
ITreeIndexAccessor[] immutableRTreeAccessors = new ITreeIndexAccessor[numImmutableComponents];
int j = 0;
for (int i = numMutableComponents; i < operationalComponents.size(); i++) {
ILSMComponent component = operationalComponents.get(i);
rangeCursors[j] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame());
RTree rtree = ((LSMRTreeDiskComponent) component).getRTree();
immutableRTreeAccessors[j] = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
immutableRTreeAccessors[j].search(rangeCursors[j], searchPred);
j++;
}
searchNextCursor();
setPriorityQueueComparator();
initPriorityQueue();
open = true;
}
use of org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor in project asterixdb by apache.
the class LSMRTree method flush.
@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent();
// Renaming order is critical because we use assume ordering when we
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
// scan the memory RTree
ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true);
//count the number of tuples in the buddy btree
ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
IIndexCursor btreeCountingCursor = ((BTreeAccessor) memBTreeAccessor).createCountingSearchCursor();
memBTreeAccessor.search(btreeCountingCursor, btreeNullPredicate);
long numBTreeTuples = 0L;
try {
while (btreeCountingCursor.hasNext()) {
btreeCountingCursor.next();
ITupleReference countTuple = btreeCountingCursor.getTuple();
numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
}
} finally {
btreeCountingCursor.close();
}
ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
ITreeIndexCursor cursor;
IBinaryComparatorFactory[] linearizerArray = { linearizer };
TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(), linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), flushingComponent.getRTree().getBufferCache(), comparatorFields);
// BulkLoad the tuples from the in-memory tree into the new disk
// RTree.
boolean isEmpty = true;
try {
while (rtreeScanCursor.hasNext()) {
isEmpty = false;
rtreeScanCursor.next();
rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
}
} finally {
rtreeScanCursor.close();
}
rTreeTupleSorter.sort();
cursor = rTreeTupleSorter;
if (!isEmpty) {
try {
while (cursor.hasNext()) {
cursor.next();
ITupleReference frameTuple = cursor.getTuple();
componentBulkLoader.add(frameTuple);
}
} finally {
cursor.close();
}
}
// scan the memory BTree
IIndexCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
try {
while (btreeScanCursor.hasNext()) {
btreeScanCursor.next();
ITupleReference frameTuple = btreeScanCursor.getTuple();
componentBulkLoader.delete(frameTuple);
}
} finally {
btreeScanCursor.close();
}
if (component.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
// Note. If we change the filter to write to metadata object, we don't need the if block above
flushingComponent.getMetadata().copy(component.getMetadata());
componentBulkLoader.end();
return component;
}
use of org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor in project asterixdb by apache.
the class LSMRTreeAbstractCursor method open.
@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
if (btreeCmp == null) {
btreeCmp = lsmInitialState.getBTreeCmp();
btreeRangePredicate.setLowKeyCmp(btreeCmp);
btreeRangePredicate.setHighKeyCmp(btreeCmp);
}
operationalComponents = lsmInitialState.getOperationalComponents();
lsmHarness = lsmInitialState.getLSMHarness();
numberOfTrees = operationalComponents.size();
int numComponenets = operationalComponents.size();
if (rtreeCursors == null || rtreeCursors.length != numComponenets) {
// object creation: should be relatively low
rtreeCursors = new RTreeSearchCursor[numberOfTrees];
btreeCursors = new BTreeRangeSearchCursor[numberOfTrees];
rtreeAccessors = new RTreeAccessor[numberOfTrees];
btreeAccessors = new BTreeAccessor[numberOfTrees];
}
includeMutableComponent = false;
for (int i = 0; i < numberOfTrees; i++) {
ILSMComponent component = operationalComponents.get(i);
RTree rtree;
BTree btree;
if (component.getType() == LSMComponentType.MEMORY) {
includeMutableComponent = true;
// No need for a bloom filter for the in-memory BTree.
if (btreeCursors[i] == null || btreeCursors[i].isBloomFilterAware()) {
//create
btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame(), false);
} else {
//re-use
btreeCursors[i].reset();
}
rtree = ((LSMRTreeMemoryComponent) component).getRTree();
btree = ((LSMRTreeMemoryComponent) component).getBTree();
} else {
if (btreeCursors[i] == null || !btreeCursors[i].isBloomFilterAware()) {
// need to create a new one
btreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame(), false, ((LSMRTreeDiskComponent) operationalComponents.get(i)).getBloomFilter());
} else {
// reset
((BloomFilterAwareBTreePointSearchCursor) btreeCursors[i]).resetBloomFilter(((LSMRTreeDiskComponent) operationalComponents.get(i)).getBloomFilter());
btreeCursors[i].reset();
}
rtree = ((LSMRTreeDiskComponent) component).getRTree();
btree = ((LSMRTreeDiskComponent) component).getBTree();
}
if (rtreeCursors[i] == null) {
rtreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame());
} else {
rtreeCursors[i].reset();
}
if (rtreeAccessors[i] == null) {
rtreeAccessors[i] = (RTreeAccessor) rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
} else {
rtreeAccessors[i].reset(rtree, NoOpOperationCallback.INSTANCE);
btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
}
}
rtreeSearchPredicate = (SearchPredicate) searchPred;
btreeRangePredicate.setHighKey(null);
btreeRangePredicate.setLowKey(null);
open = true;
}
use of org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor in project asterixdb by apache.
the class LSMRTreeWithAntiMatterTuples method flush.
@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
// Renaming order is critical because we use assume ordering when we
// read the file names when we open the tree.
// The RTree should be renamed before the BTree.
LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent();
ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
// Since the LSM-RTree is used as a secondary assumption, the
// primary key will be the last comparator in the BTree comparators
TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(), linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), flushingComponent.getRTree().getBufferCache(), comparatorFields);
boolean isEmpty = true;
try {
while (rtreeScanCursor.hasNext()) {
isEmpty = false;
rtreeScanCursor.next();
rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
}
} finally {
rtreeScanCursor.close();
}
if (!isEmpty) {
rTreeTupleSorter.sort();
}
// scan the memory BTree
ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
BTreeRangeSearchCursor btreeScanCursor = (BTreeRangeSearchCursor) memBTreeAccessor.createSearchCursor(false);
RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBTree().getFileId(), linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(), flushingComponent.getBTree().getBufferCache(), comparatorFields);
isEmpty = true;
try {
while (btreeScanCursor.hasNext()) {
isEmpty = false;
btreeScanCursor.next();
bTreeTupleSorter.insertTupleEntry(btreeScanCursor.getPageId(), btreeScanCursor.getTupleOffset());
}
} finally {
btreeScanCursor.close();
}
if (!isEmpty) {
bTreeTupleSorter.sort();
}
LSMRTreeWithAntiMatterTuplesFlushCursor cursor = new LSMRTreeWithAntiMatterTuplesFlushCursor(rTreeTupleSorter, bTreeTupleSorter, comparatorFields, linearizerArray);
cursor.open(null, null);
try {
while (cursor.hasNext()) {
cursor.next();
ITupleReference frameTuple = cursor.getTuple();
componentBulkLoader.add(frameTuple);
}
} finally {
cursor.close();
}
if (component.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<>();
filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
flushingComponent.getMetadata().copy(component.getMetadata());
componentBulkLoader.end();
return component;
}
use of org.apache.hyracks.storage.am.rtree.impls.RTreeSearchCursor in project asterixdb by apache.
the class RTreeSearchCursorTest method rangeSearchTest.
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void rangeSearchTest() throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("TESTING RANGE SEARCH CURSOR FOR RTREE");
}
IBufferCache bufferCache = harness.getBufferCache();
// Declare fields.
int fieldCount = 5;
ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
typeTraits[0] = IntegerPointable.TYPE_TRAITS;
typeTraits[1] = IntegerPointable.TYPE_TRAITS;
typeTraits[2] = IntegerPointable.TYPE_TRAITS;
typeTraits[3] = IntegerPointable.TYPE_TRAITS;
typeTraits[4] = IntegerPointable.TYPE_TRAITS;
// Declare field serdes.
ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
// Declare keys.
int keyFieldCount = 4;
IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
cmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// create value providers
IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(cmpFactories.length, IntegerPointable.FACTORY);
RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory, valueProviderFactories, RTreePolicyType.RTREE, false);
ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory, valueProviderFactories, RTreePolicyType.RTREE, false);
IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
RTree rtree = new RTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference(), false);
rtree.create();
rtree.activate();
ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
ArrayTupleReference tuple = new ArrayTupleReference();
ITreeIndexAccessor indexAccessor = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
int numInserts = 10000;
ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<>();
for (int i = 0; i < numInserts; i++) {
int p1x = rnd.nextInt();
int p1y = rnd.nextInt();
int p2x = rnd.nextInt();
int p2y = rnd.nextInt();
int pk = rnd.nextInt();
;
TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), pk);
try {
indexAccessor.insert(tuple);
} catch (HyracksDataException e) {
if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
throw e;
}
}
RTreeCheckTuple checkTuple = new RTreeCheckTuple(fieldCount, keyFieldCount);
checkTuple.appendField(Math.min(p1x, p2x));
checkTuple.appendField(Math.min(p1y, p2y));
checkTuple.appendField(Math.max(p1x, p2x));
checkTuple.appendField(Math.max(p1y, p2y));
checkTuple.appendField(pk);
checkTuples.add(checkTuple);
}
// Build key.
ArrayTupleBuilder keyTb = new ArrayTupleBuilder(keyFieldCount);
ArrayTupleReference key = new ArrayTupleReference();
TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
MultiComparator cmp = MultiComparator.create(cmpFactories);
ITreeIndexCursor searchCursor = new RTreeSearchCursor(interiorFrame, leafFrame);
SearchPredicate searchPredicate = new SearchPredicate(key, cmp);
RTreeCheckTuple keyCheck = (RTreeCheckTuple) rTreeTestUtils.createCheckTupleFromTuple(key, fieldSerdes, keyFieldCount);
HashMultiSet<RTreeCheckTuple> expectedResult = rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
indexAccessor.search(searchCursor, searchPredicate);
rTreeTestUtils.checkExpectedResults(searchCursor, expectedResult, fieldSerdes, keyFieldCount, null);
rtree.deactivate();
rtree.destroy();
}
Aggregations