Examples with IIndexCursor org.apache.hyracks.storage.common.IIndexCursor used on opensource projects

Example 21 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMInvertedIndexTestWorker method performOp.

@Override
public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
    LSMInvertedIndexAccessor accessor = (LSMInvertedIndexAccessor) indexAccessor;
    IIndexCursor searchCursor = accessor.createSearchCursor(false);
    IIndexCursor rangeSearchCursor = accessor.createRangeSearchCursor();
    RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
    IBinaryTokenizerFactory tokenizerFactory = invIndex.getTokenizerFactory();
    int searchModifierIndex = Math.abs(rnd.nextInt()) % TEST_SEARCH_MODIFIERS.length;
    InvertedIndexSearchPredicate searchPred = new InvertedIndexSearchPredicate(tokenizerFactory.createTokenizer(), TEST_SEARCH_MODIFIERS[searchModifierIndex]);
    switch(op) {
        case INSERT:
            {
                insert(accessor, tuple);
                break;
            }
        case DELETE:
            {
                // Randomly pick a document from the corpus to delete.
                if (!documentCorpus.isEmpty()) {
                    int docIndex = Math.abs(rnd.nextInt()) % documentCorpus.size();
                    ITupleReference deleteTuple = documentCorpus.get(docIndex);
                    accessor.delete(deleteTuple);
                    // Swap tupleIndex with last element.
                    documentCorpus.set(docIndex, documentCorpus.get(documentCorpus.size() - 1));
                    documentCorpus.remove(documentCorpus.size() - 1);
                } else {
                    // No existing documents to delete, treat this case as an insert.
                    insert(accessor, tuple);
                }
                break;
            }
        case POINT_SEARCH:
            {
                searchCursor.reset();
                searchPred.setQueryTuple(tuple);
                searchPred.setQueryFieldIndex(0);
                try {
                    accessor.search(searchCursor, searchPred);
                    consumeCursorTuples(searchCursor);
                } catch (HyracksDataException e) {
                    // Ignore.
                    if (e.getErrorCode() != ErrorCode.OCCURRENCE_THRESHOLD_PANIC_EXCEPTION) {
                        throw e;
                    }
                }
                break;
            }
        case SCAN:
            {
                rangeSearchCursor.reset();
                accessor.rangeSearch(rangeSearchCursor, rangePred);
                consumeCursorTuples(rangeSearchCursor);
                break;
            }
        case MERGE:
            {
                accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), invIndex.getImmutableComponents());
                break;
            }
        default:
            throw new HyracksDataException("Op " + op.toString() + " not supported.");
    }
}

Example 22 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class ExternalRTree method merge.

// The only reason for overriding the merge method is the way to determine
// the need to keep deleted tuples
// This can be done in a better way by creating a method boolean
// keepDeletedTuples(mergedComponents);
@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
    ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rtreeSearchPred);
    LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
    // In case we must keep the deleted-keys BTrees, then they must be
    // merged *before* merging the r-trees so that
    // lsmHarness.endSearch() is called once when the r-trees have been
    // merged.
    boolean keepDeleteTuples = false;
    if (version == 0) {
        keepDeleteTuples = mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1);
    } else {
        keepDeleteTuples = mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != secondDiskComponents.get(secondDiskComponents.size() - 1);
    }
    if (keepDeleteTuples) {
        // Keep the deleted tuples since the oldest disk component is not
        // included in the merge operation
        LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx);
        search(opCtx, btreeCursor, rtreeSearchPred);
        BTree btree = mergedComponent.getBTree();
        IIndexBulkLoader btreeBulkLoader = btree.createBulkLoader(1.0f, true, 0L, false);
        long numElements = 0L;
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
        }
        int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElements);
        BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement, bloomFilterFalsePositiveRate);
        IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements, bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
        try {
            while (btreeCursor.hasNext()) {
                btreeCursor.next();
                ITupleReference tuple = btreeCursor.getTuple();
                btreeBulkLoader.add(tuple);
                builder.add(tuple);
            }
        } finally {
            btreeCursor.close();
            builder.end();
        }
        btreeBulkLoader.end();
    }
    IIndexBulkLoader bulkLoader = mergedComponent.getRTree().createBulkLoader(1.0f, false, 0L, false);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            bulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    bulkLoader.end();
    return mergedComponent;
}

Example 23 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class ExternalBTreeWithBuddy method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeWithBuddyMergeOperation mergeOp = (LSMBTreeWithBuddyMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    ISearchPredicate btreeSearchPred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMBTreeWithBuddySortedCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, btreeSearchPred);
    LSMBTreeWithBuddyDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBuddyBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
    IIndexBulkLoader componentBulkLoader;
    if (mergeOp.isKeepDeletedTuples()) {
        // Keep the deleted tuples since the oldest disk component is not
        // included in the merge operation
        LSMBuddyBTreeMergeCursor buddyBtreeCursor = new LSMBuddyBTreeMergeCursor(opCtx);
        search(opCtx, buddyBtreeCursor, btreeSearchPred);
        long numElements = 0L;
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            numElements += ((LSMBTreeWithBuddyDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
        }
        componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
        try {
            while (buddyBtreeCursor.hasNext()) {
                buddyBtreeCursor.next();
                ITupleReference tuple = buddyBtreeCursor.getTuple();
                ((LSMBTreeWithBuddyDiskComponentBulkLoader) componentBulkLoader).delete(tuple);
            }
        } finally {
            buddyBtreeCursor.close();
        }
    } else {
        componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false);
    }
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    componentBulkLoader.end();
    return mergedComponent;
}