Search in sources :

Example 11 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class RTreeTestWorker method performOp.

@Override
public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
    RTree.RTreeAccessor accessor = (RTree.RTreeAccessor) indexAccessor;
    IIndexCursor searchCursor = accessor.createSearchCursor(false);
    ITreeIndexCursor diskOrderScanCursor = accessor.createDiskOrderScanCursor();
    MultiComparator cmp = accessor.getOpContext().getCmp();
    SearchPredicate rangePred = new SearchPredicate(tuple, cmp);
    switch(op) {
        case INSERT:
            rearrangeTuple(tuple, cmp);
            accessor.insert(rearrangedTuple);
            break;
        case DELETE:
            rearrangeTuple(tuple, cmp);
            accessor.delete(rearrangedTuple);
            break;
        case SCAN:
            searchCursor.reset();
            rangePred.setSearchKey(null);
            accessor.search(searchCursor, rangePred);
            consumeCursorTuples(searchCursor);
            break;
        case DISKORDER_SCAN:
            diskOrderScanCursor.reset();
            accessor.diskOrderScan(diskOrderScanCursor);
            consumeCursorTuples(diskOrderScanCursor);
            break;
        default:
            throw new HyracksDataException("Op " + op.toString() + " not supported.");
    }
}
Also used : ITreeIndexCursor(org.apache.hyracks.storage.am.common.api.ITreeIndexCursor) MultiComparator(org.apache.hyracks.storage.common.MultiComparator) IIndexCursor(org.apache.hyracks.storage.common.IIndexCursor) RTree(org.apache.hyracks.storage.am.rtree.impls.RTree) SearchPredicate(org.apache.hyracks.storage.am.rtree.impls.SearchPredicate) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException)

Example 12 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMBTree method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
    LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent();
    IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        IIndexCursor countingCursor = ((BTreeAccessor) accessor).createCountingSearchCursor();
        accessor.search(countingCursor, nullPred);
        try {
            while (countingCursor.hasNext()) {
                countingCursor.next();
                ITupleReference countTuple = countingCursor.getTuple();
                numElements = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
            }
        } finally {
            countingCursor.close();
        }
    }
    LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
    IIndexCursor scanCursor = accessor.createSearchCursor(false);
    accessor.search(scanCursor, nullPred);
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            componentBulkLoader.add(scanCursor.getTuple());
        }
    } finally {
        scanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getBTree());
    }
    // Write metadata from memory component to disk
    // Q. what about the merge operation? how do we resolve conflicts
    // A. Through providing an appropriate ILSMIOOperationCallback
    // Must not reset the metadata before the flush is completed
    // Use the copy of the metadata in the opContext
    // TODO This code should be in the callback and not in the index
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}
Also used : RangePredicate(org.apache.hyracks.storage.am.btree.impls.RangePredicate) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) ArrayList(java.util.ArrayList) IIndexCursor(org.apache.hyracks.storage.common.IIndexCursor) BTreeAccessor(org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor) ILSMDiskComponentBulkLoader(org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader) IIndexAccessor(org.apache.hyracks.storage.common.IIndexAccessor)

Example 13 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMBTree method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rangePred);
    List<ILSMComponent> mergedComponents = mergeOp.getMergingComponents();
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        for (int i = 0; i < mergedComponents.size(); ++i) {
            numElements += ((LSMBTreeDiskComponent) mergedComponents.get(i)).getBloomFilter().getNumElements();
        }
    }
    LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (mergedComponent.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
    }
    componentBulkLoader.end();
    return mergedComponent;
}
Also used : RangePredicate(org.apache.hyracks.storage.am.btree.impls.RangePredicate) ArrayList(java.util.ArrayList) ILSMIndexOperationContext(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext) LSMIndexSearchCursor(org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor) ILSMDiskComponentBulkLoader(org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader) ILSMComponent(org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) IIndexCursor(org.apache.hyracks.storage.common.IIndexCursor)

Example 14 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMBTree method insert.

private boolean insert(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException {
    LSMBTreePointSearchCursor searchCursor = ctx.getInsertSearchCursor();
    IIndexCursor memCursor = ctx.getMemCursor();
    RangePredicate predicate = (RangePredicate) ctx.getSearchPredicate();
    predicate.setHighKey(tuple);
    predicate.setLowKey(tuple);
    if (needKeyDupCheck) {
        // first check the inmemory component
        ctx.getCurrentMutableBTreeAccessor().search(memCursor, predicate);
        try {
            if (memCursor.hasNext()) {
                memCursor.next();
                LSMBTreeTupleReference lsmbtreeTuple = (LSMBTreeTupleReference) memCursor.getTuple();
                if (!lsmbtreeTuple.isAntimatter()) {
                    throw HyracksDataException.create(ErrorCode.DUPLICATE_KEY);
                } else {
                    memCursor.close();
                    ctx.getCurrentMutableBTreeAccessor().upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
                    return true;
                }
            }
        } finally {
            memCursor.close();
        }
        // TODO: Can we just remove the above code that search the mutable
        // component and do it together with the search call below? i.e. instead
        // of passing false to the lsmHarness.search(), we pass true to include
        // the mutable component?
        // the key was not in the inmemory component, so check the disk
        // components
        // This is a hack to avoid searching the current active mutable component twice. It is critical to add it back once the search is over.
        ILSMComponent firstComponent = ctx.getComponentHolder().remove(0);
        search(ctx, searchCursor, predicate);
        try {
            if (searchCursor.hasNext()) {
                throw HyracksDataException.create(ErrorCode.DUPLICATE_KEY);
            }
        } finally {
            searchCursor.close();
            // Add the current active mutable component back
            ctx.getComponentHolder().add(0, firstComponent);
        }
    }
    ctx.getCurrentMutableBTreeAccessor().upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
    return true;
}
Also used : RangePredicate(org.apache.hyracks.storage.am.btree.impls.RangePredicate) LSMBTreeTupleReference(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleReference) ILSMComponent(org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent) IIndexCursor(org.apache.hyracks.storage.common.IIndexCursor)

Example 15 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMInvertedIndexTestUtils method compareActualAndExpectedIndexesRangeSearch.

/**
     * Compares actual and expected indexes using the rangeSearch() method of the inverted-index accessor.
     */
public static void compareActualAndExpectedIndexesRangeSearch(LSMInvertedIndexTestContext testCtx) throws HyracksDataException {
    IInvertedIndex invIndex = (IInvertedIndex) testCtx.getIndex();
    int tokenFieldCount = invIndex.getTokenTypeTraits().length;
    int invListFieldCount = invIndex.getInvListTypeTraits().length;
    IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) invIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor invIndexCursor = invIndexAccessor.createRangeSearchCursor();
    MultiComparator tokenCmp = MultiComparator.create(invIndex.getTokenCmpFactories());
    IBinaryComparatorFactory[] tupleCmpFactories = new IBinaryComparatorFactory[tokenFieldCount + invListFieldCount];
    for (int i = 0; i < tokenFieldCount; i++) {
        tupleCmpFactories[i] = invIndex.getTokenCmpFactories()[i];
    }
    for (int i = 0; i < invListFieldCount; i++) {
        tupleCmpFactories[tokenFieldCount + i] = invIndex.getInvListCmpFactories()[i];
    }
    MultiComparator tupleCmp = MultiComparator.create(tupleCmpFactories);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, tokenCmp, tokenCmp);
    invIndexAccessor.rangeSearch(invIndexCursor, nullPred);
    // Helpers for generating a serialized inverted-list element from a CheckTuple from the expected index.
    ISerializerDeserializer[] fieldSerdes = testCtx.getFieldSerdes();
    ArrayTupleBuilder expectedBuilder = new ArrayTupleBuilder(fieldSerdes.length);
    ArrayTupleReference expectedTuple = new ArrayTupleReference();
    Iterator<CheckTuple> expectedIter = testCtx.getCheckTuples().iterator();
    // Compare index elements.
    try {
        while (invIndexCursor.hasNext() && expectedIter.hasNext()) {
            invIndexCursor.next();
            ITupleReference actualTuple = invIndexCursor.getTuple();
            CheckTuple expected = expectedIter.next();
            OrderedIndexTestUtils.createTupleFromCheckTuple(expected, expectedBuilder, expectedTuple, fieldSerdes);
            if (tupleCmp.compare(actualTuple, expectedTuple) != 0) {
                fail("Index entries differ for token '" + expected.getField(0) + "'.");
            }
        }
        if (expectedIter.hasNext()) {
            fail("Indexes do not match. Actual index is missing entries.");
        }
        if (invIndexCursor.hasNext()) {
            fail("Indexes do not match. Actual index contains too many entries.");
        }
    } finally {
        invIndexCursor.close();
    }
}
Also used : RangePredicate(org.apache.hyracks.storage.am.btree.impls.RangePredicate) MultiComparator(org.apache.hyracks.storage.common.MultiComparator) ArrayTupleReference(org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) IInvertedIndexAccessor(org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor) ArrayTupleBuilder(org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder) ISerializerDeserializer(org.apache.hyracks.api.dataflow.value.ISerializerDeserializer) CheckTuple(org.apache.hyracks.storage.am.common.CheckTuple) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) IIndexCursor(org.apache.hyracks.storage.common.IIndexCursor) IInvertedIndex(org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex)

Aggregations

IIndexCursor (org.apache.hyracks.storage.common.IIndexCursor)23 ITupleReference (org.apache.hyracks.dataflow.common.data.accessors.ITupleReference)18 RangePredicate (org.apache.hyracks.storage.am.btree.impls.RangePredicate)15 ArrayList (java.util.ArrayList)8 HyracksDataException (org.apache.hyracks.api.exceptions.HyracksDataException)8 ILSMDiskComponentBulkLoader (org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader)7 ILSMIndexOperationContext (org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext)6 MultiComparator (org.apache.hyracks.storage.common.MultiComparator)6 SearchPredicate (org.apache.hyracks.storage.am.rtree.impls.SearchPredicate)5 IIndexAccessor (org.apache.hyracks.storage.common.IIndexAccessor)5 ISearchPredicate (org.apache.hyracks.storage.common.ISearchPredicate)5 ArrayTupleBuilder (org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder)3 ArrayTupleReference (org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference)3 BTreeAccessor (org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor)3 CheckTuple (org.apache.hyracks.storage.am.common.CheckTuple)3 LSMIndexSearchCursor (org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor)3 IInvertedIndexAccessor (org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor)3 IBinaryComparatorFactory (org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory)2 ITreeIndexCursor (org.apache.hyracks.storage.am.common.api.ITreeIndexCursor)2 PermutingTupleReference (org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference)2