Examples with ILSMComponent org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent used on opensource projects

Example 16 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class LSMRTreeAbstractCursor method open.

@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
    LSMRTreeCursorInitialState lsmInitialState = (LSMRTreeCursorInitialState) initialState;
    if (btreeCmp == null) {
        btreeCmp = lsmInitialState.getBTreeCmp();
        btreeRangePredicate.setLowKeyCmp(btreeCmp);
        btreeRangePredicate.setHighKeyCmp(btreeCmp);
    }
    operationalComponents = lsmInitialState.getOperationalComponents();
    lsmHarness = lsmInitialState.getLSMHarness();
    numberOfTrees = operationalComponents.size();
    int numComponenets = operationalComponents.size();
    if (rtreeCursors == null || rtreeCursors.length != numComponenets) {
        // object creation: should be relatively low
        rtreeCursors = new RTreeSearchCursor[numberOfTrees];
        btreeCursors = new BTreeRangeSearchCursor[numberOfTrees];
        rtreeAccessors = new RTreeAccessor[numberOfTrees];
        btreeAccessors = new BTreeAccessor[numberOfTrees];
    }
    includeMutableComponent = false;
    for (int i = 0; i < numberOfTrees; i++) {
        ILSMComponent component = operationalComponents.get(i);
        RTree rtree;
        BTree btree;
        if (component.getType() == LSMComponentType.MEMORY) {
            includeMutableComponent = true;
            // No need for a bloom filter for the in-memory BTree.
            if (btreeCursors[i] == null || btreeCursors[i].isBloomFilterAware()) {
                //create
                btreeCursors[i] = new BTreeRangeSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame(), false);
            } else {
                //re-use
                btreeCursors[i].reset();
            }
            rtree = ((LSMRTreeMemoryComponent) component).getRTree();
            btree = ((LSMRTreeMemoryComponent) component).getBTree();
        } else {
            if (btreeCursors[i] == null || !btreeCursors[i].isBloomFilterAware()) {
                // need to create a new one
                btreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor((IBTreeLeafFrame) lsmInitialState.getBTreeLeafFrameFactory().createFrame(), false, ((LSMRTreeDiskComponent) operationalComponents.get(i)).getBloomFilter());
            } else {
                // reset
                ((BloomFilterAwareBTreePointSearchCursor) btreeCursors[i]).resetBloomFilter(((LSMRTreeDiskComponent) operationalComponents.get(i)).getBloomFilter());
                btreeCursors[i].reset();
            }
            rtree = ((LSMRTreeDiskComponent) component).getRTree();
            btree = ((LSMRTreeDiskComponent) component).getBTree();
        }
        if (rtreeCursors[i] == null) {
            rtreeCursors[i] = new RTreeSearchCursor((IRTreeInteriorFrame) lsmInitialState.getRTreeInteriorFrameFactory().createFrame(), (IRTreeLeafFrame) lsmInitialState.getRTreeLeafFrameFactory().createFrame());
        } else {
            rtreeCursors[i].reset();
        }
        if (rtreeAccessors[i] == null) {
            rtreeAccessors[i] = (RTreeAccessor) rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            btreeAccessors[i] = (BTreeAccessor) btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        } else {
            rtreeAccessors[i].reset(rtree, NoOpOperationCallback.INSTANCE);
            btreeAccessors[i].reset(btree, NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        }
    }
    rtreeSearchPredicate = (SearchPredicate) searchPred;
    btreeRangePredicate.setHighKey(null);
    btreeRangePredicate.setLowKey(null);
    open = true;
}

Example 17 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class LSMInvertedIndex method search.

@Override
public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred) throws HyracksDataException {
    List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
    int numComponents = operationalComponents.size();
    boolean includeMutableComponent = false;
    ArrayList<IIndexAccessor> indexAccessors = new ArrayList<>(numComponents);
    ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<>(numComponents);
    for (int i = 0; i < operationalComponents.size(); i++) {
        ILSMComponent component = operationalComponents.get(i);
        if (component.getType() == LSMComponentType.MEMORY) {
            includeMutableComponent = true;
            IIndexAccessor invIndexAccessor = ((LSMInvertedIndexMemoryComponent) component).getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            indexAccessors.add(invIndexAccessor);
            IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMemoryComponent) component).getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            deletedKeysBTreeAccessors.add(deletedKeysAccessor);
        } else {
            IIndexAccessor invIndexAccessor = ((LSMInvertedIndexDiskComponent) component).getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            indexAccessors.add(invIndexAccessor);
            IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexDiskComponent) component).getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            deletedKeysBTreeAccessors.add(deletedKeysAccessor);
        }
    }
    ICursorInitialState initState = createCursorInitialState(pred, ictx, includeMutableComponent, indexAccessors, deletedKeysBTreeAccessors, operationalComponents);
    cursor.open(initState, pred);
}

Example 18 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class LSMInvertedIndexRangeSearchCursor method open.

@Override
public void open(ICursorInitialState initState, ISearchPredicate searchPred) throws HyracksDataException {
    LSMInvertedIndexRangeSearchCursorInitialState lsmInitState = (LSMInvertedIndexRangeSearchCursorInitialState) initState;
    cmp = lsmInitState.getOriginalKeyComparator();
    int numComponents = lsmInitState.getNumComponents();
    rangeCursors = new IIndexCursor[numComponents];
    for (int i = 0; i < numComponents; i++) {
        IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) lsmInitState.getIndexAccessors().get(i);
        rangeCursors[i] = invIndexAccessor.createRangeSearchCursor();
        invIndexAccessor.rangeSearch(rangeCursors[i], lsmInitState.getSearchPredicate());
    }
    lsmHarness = lsmInitState.getLSMHarness();
    operationalComponents = lsmInitState.getOperationalComponents();
    includeMutableComponent = lsmInitState.getIncludeMemComponent();
    // For searching the deleted-keys BTrees.
    this.keysOnlyTuple = lsmInitState.getKeysOnlyTuple();
    deletedKeysBTreeAccessors = lsmInitState.getDeletedKeysBTreeAccessors();
    if (!deletedKeysBTreeAccessors.isEmpty()) {
        deletedKeysBTreeCursors = new IIndexCursor[deletedKeysBTreeAccessors.size()];
        for (int i = 0; i < operationalComponents.size(); i++) {
            ILSMComponent component = operationalComponents.get(i);
            if (component.getType() == LSMComponentType.MEMORY) {
                // No need for a bloom filter for the in-memory BTree.
                deletedKeysBTreeCursors[i] = deletedKeysBTreeAccessors.get(i).createSearchCursor(false);
            } else {
                deletedKeysBTreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor((IBTreeLeafFrame) lsmInitState.getgetDeletedKeysBTreeLeafFrameFactory().createFrame(), false, ((LSMInvertedIndexDiskComponent) operationalComponents.get(i)).getBloomFilter());
            }
        }
    }
    MultiComparator keyCmp = lsmInitState.getKeyComparator();
    keySearchPred = new RangePredicate(keysOnlyTuple, keysOnlyTuple, true, true, keyCmp, keyCmp);
    setPriorityQueueComparator();
    initPriorityQueue();
}

Example 19 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class ExternalRTree method scheduleMerge.

// The only change the the schedule merge is the method used to create the
// opCtx. first line <- in schedule merge, we->
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
    ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, -1);
    rctx.setOperation(IndexOperation.MERGE);
    List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
    ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
    LSMComponentFileReferences relMergeFileRefs = getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(0), (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1));
    ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), rctx, buddyBTreeFields);
    // create the merge operation.
    LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
    ioScheduler.scheduleOperation(mergeOp);
}

Example 20 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class LSMBTree method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rangePred);
    List<ILSMComponent> mergedComponents = mergeOp.getMergingComponents();
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        for (int i = 0; i < mergedComponents.size(); ++i) {
            numElements += ((LSMBTreeDiskComponent) mergedComponents.get(i)).getBloomFilter().getNumElements();
        }
    }
    LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (mergedComponent.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
    }
    componentBulkLoader.end();
    return mergedComponent;
}