Examples with ITreeIndexAccessor org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor used on opensource projects

Example 1 with ITreeIndexAccessor

use of org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor in project asterixdb by apache.

the class LSMRTreeWithAntiMatterTuplesSearchCursor method open.

@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
    LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
    cmp = lsmInitialState.getHilbertCmp();
    btreeCmp = lsmInitialState.getBTreeCmp();
    lsmHarness = lsmInitialState.getLSMHarness();
    comparatorFields = lsmInitialState.getComparatorFields();
    operationalComponents = lsmInitialState.getOperationalComponents();
    rtreeSearchPredicate = (SearchPredicate) searchPred;
    searchCallback = lsmInitialState.getSearchOperationCallback();
    includeMutableComponent = false;
    numMutableComponents = 0;
    int numImmutableComponents = 0;
    for (ILSMComponent component : operationalComponents) {
        if (component.getType() == LSMComponentType.MEMORY) {
            includeMutableComponent = true;
            numMutableComponents++;
        } else {
            numImmutableComponents++;
        }
    }
    if (includeMutableComponent) {
        btreeRangePredicate = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
    }
    mutableRTreeCursors = new RTreeSearchCursor[numMutableComponents];
    mutableRTreeAccessors = new ITreeIndexAccessor[numMutableComponents];
    btreeCursors = new BTreeRangeSearchCursor[numMutableComponents];
    btreeAccessors = new ITreeIndexAccessor[numMutableComponents];
    for (int i = 0; i < numMutableComponents; i++) {
        ILSMComponent component = operationalComponents.get(i);
        RTree rtree = ((LSMRTreeMemoryComponent) component).getRTree();
        BTree btree = ((LSMRTreeMemoryComponent) component).getBTree();
        mutableRTreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame());
        btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame(), false);
        btreeAccessors[i] = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        mutableRTreeAccessors[i] = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    }
    rangeCursors = new RTreeSearchCursor[numImmutableComponents];
    ITreeIndexAccessor[] immutableRTreeAccessors = new ITreeIndexAccessor[numImmutableComponents];
    int j = 0;
    for (int i = numMutableComponents; i < operationalComponents.size(); i++) {
        ILSMComponent component = operationalComponents.get(i);
        rangeCursors[j] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame());
        RTree rtree = ((LSMRTreeDiskComponent) component).getRTree();
        immutableRTreeAccessors[j] = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        immutableRTreeAccessors[j].search(rangeCursors[j], searchPred);
        j++;
    }
    searchNextCursor();
    setPriorityQueueComparator();
    initPriorityQueue();
    open = true;
}

Example 2 with ITreeIndexAccessor

use of org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor in project asterixdb by apache.

the class LSMRTree method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
    LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent();
    // Renaming order is critical because we use assume ordering when we
    // read the file names when we open the tree.
    // The RTree should be renamed before the BTree.
    // scan the memory RTree
    ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
    SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
    memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
    LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true);
    //count the number of tuples in the buddy btree
    ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
    IIndexCursor btreeCountingCursor = ((BTreeAccessor) memBTreeAccessor).createCountingSearchCursor();
    memBTreeAccessor.search(btreeCountingCursor, btreeNullPredicate);
    long numBTreeTuples = 0L;
    try {
        while (btreeCountingCursor.hasNext()) {
            btreeCountingCursor.next();
            ITupleReference countTuple = btreeCountingCursor.getTuple();
            numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
        }
    } finally {
        btreeCountingCursor.close();
    }
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
    ITreeIndexCursor cursor;
    IBinaryComparatorFactory[] linearizerArray = { linearizer };
    TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(), linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), flushingComponent.getRTree().getBufferCache(), comparatorFields);
    // BulkLoad the tuples from the in-memory tree into the new disk
    // RTree.
    boolean isEmpty = true;
    try {
        while (rtreeScanCursor.hasNext()) {
            isEmpty = false;
            rtreeScanCursor.next();
            rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
        }
    } finally {
        rtreeScanCursor.close();
    }
    rTreeTupleSorter.sort();
    cursor = rTreeTupleSorter;
    if (!isEmpty) {
        try {
            while (cursor.hasNext()) {
                cursor.next();
                ITupleReference frameTuple = cursor.getTuple();
                componentBulkLoader.add(frameTuple);
            }
        } finally {
            cursor.close();
        }
    }
    // scan the memory BTree
    IIndexCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
    memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
    try {
        while (btreeScanCursor.hasNext()) {
            btreeScanCursor.next();
            ITupleReference frameTuple = btreeScanCursor.getTuple();
            componentBulkLoader.delete(frameTuple);
        }
    } finally {
        btreeScanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
    }
    // Note. If we change the filter to write to metadata object, we don't need the if block above
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}

Example 3 with ITreeIndexAccessor

use of org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor in project asterixdb by apache.

the class OrderedIndexExamplesTest method diskOrderScan.

protected void diskOrderScan(IIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes) throws Exception {
    try {
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("Disk-Order Scan:");
        }
        ITreeIndexAccessor treeIndexAccessor = (ITreeIndexAccessor) indexAccessor;
        TreeIndexDiskOrderScanCursor diskOrderCursor = (TreeIndexDiskOrderScanCursor) treeIndexAccessor.createDiskOrderScanCursor();
        treeIndexAccessor.diskOrderScan(diskOrderCursor);
        try {
            while (diskOrderCursor.hasNext()) {
                diskOrderCursor.next();
                ITupleReference frameTuple = diskOrderCursor.getTuple();
                String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
                if (LOGGER.isLoggable(Level.INFO)) {
                    LOGGER.info(rec);
                }
            }
        } finally {
            diskOrderCursor.close();
        }
    } catch (UnsupportedOperationException e) {
        // support disk-order scan.
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("Ignoring disk-order scan since it's not supported.");
        }
    } catch (ClassCastException e) {
        // an ITreeIndexAccessor, e.g., for the LSMBTree.
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("Ignoring disk-order scan since it's not supported.");
        }
    }
}

Example 4 with ITreeIndexAccessor

use of org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor in project asterixdb by apache.

the class BTreeSearchCursorTest method nonUniqueIndexTest.

@Test
public void nonUniqueIndexTest() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
    }
    IBufferCache bufferCache = harness.getBufferCache();
    // declare keys
    int keyFieldCount = 2;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // generate keys
    int numKeys = 50;
    int maxKey = 10;
    ArrayList<Integer> keys = new ArrayList<>();
    for (int i = 0; i < numKeys; i++) {
        int k = rnd.nextInt() % maxKey;
        keys.add(k);
    }
    Collections.sort(keys);
    // insert keys into btree
    for (int i = 0; i < keys.size(); i++) {
        TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
        tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
        try {
            indexAccessor.insert(tuple);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    int minSearchKey = -100;
    int maxSearchKey = 100;
    // forward searches
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
    btree.deactivate();
    btree.destroy();
}

Example 5 with ITreeIndexAccessor

use of org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor in project asterixdb by apache.

the class BTreeSearchCursorTest method uniqueIndexTest.

@Test
public void uniqueIndexTest() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
    }
    IBufferCache bufferCache = harness.getBufferCache();
    // declare keys
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // generate keys
    int numKeys = 50;
    int maxKey = 1000;
    TreeSet<Integer> uniqueKeys = new TreeSet<>();
    ArrayList<Integer> keys = new ArrayList<>();
    while (uniqueKeys.size() < numKeys) {
        int key = rnd.nextInt() % maxKey;
        uniqueKeys.add(key);
    }
    for (Integer i : uniqueKeys) {
        keys.add(i);
    }
    // insert keys into btree
    for (int i = 0; i < keys.size(); i++) {
        TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
        tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
        try {
            indexAccessor.insert(tuple);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    int minSearchKey = -100;
    int maxSearchKey = 100;
    // forward searches
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
    btree.deactivate();
    btree.destroy();
}