Examples with IIndexCursor org.apache.hyracks.storage.common.IIndexCursor used on opensource projects

Example 6 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class OrderedIndexTestUtils method checkPointSearches.

public void checkPointSearches(IIndexTestContext ictx) throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Testing Point Searches On All Expected Keys.");
    }
    OrderedIndexTestContext ctx = (OrderedIndexTestContext) ictx;
    IIndexCursor searchCursor = ctx.getIndexAccessor().createSearchCursor(false);
    ArrayTupleBuilder lowKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
    ArrayTupleReference lowKey = new ArrayTupleReference();
    ArrayTupleBuilder highKeyBuilder = new ArrayTupleBuilder(ctx.getKeyFieldCount());
    ArrayTupleReference highKey = new ArrayTupleReference();
    RangePredicate rangePred = new RangePredicate(lowKey, highKey, true, true, null, null);
    // BTree to verify the tuple can be reached.
    for (CheckTuple checkTuple : ctx.getCheckTuples()) {
        createTupleFromCheckTuple(checkTuple, lowKeyBuilder, lowKey, ctx.getFieldSerdes());
        createTupleFromCheckTuple(checkTuple, highKeyBuilder, highKey, ctx.getFieldSerdes());
        MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), lowKey);
        MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(ctx.getComparatorFactories(), highKey);
        rangePred.setLowKey(lowKey, true);
        rangePred.setHighKey(highKey, true);
        rangePred.setLowKeyComparator(lowKeyCmp);
        rangePred.setHighKeyComparator(highKeyCmp);
        ctx.getIndexAccessor().search(searchCursor, rangePred);
        try {
            // We expect exactly one answer.
            if (searchCursor.hasNext()) {
                searchCursor.next();
                ITupleReference tuple = searchCursor.getTuple();
                compareActualAndExpected(tuple, checkTuple, ctx.getFieldSerdes());
            }
            if (searchCursor.hasNext()) {
                fail("Point search returned more than one answer.");
            }
        } finally {
            searchCursor.close();
        }
    }
}

Example 7 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMInvertedIndex method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
    search(opCtx, cursor, mergePred);
    // Create an inverted index instance.
    LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(), mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader;
    // lsmHarness.endSearch() is called once when the inverted indexes have been merged.
    if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
        // Keep the deleted tuples since the oldest disk component is not included in the merge operation
        LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
        search(opCtx, btreeCursor, mergePred);
        long numElements = 0L;
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
        }
        componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
        try {
            while (btreeCursor.hasNext()) {
                btreeCursor.next();
                ITupleReference tuple = btreeCursor.getTuple();
                componentBulkLoader.delete(tuple);
            }
        } finally {
            btreeCursor.close();
        }
    } else {
        componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
    }
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference tuple = cursor.getTuple();
            componentBulkLoader.add(tuple);
        }
    } finally {
        cursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
            ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
            if (min != null) {
                filterTuples.add(min);
            }
            if (max != null) {
                filterTuples.add(max);
            }
        }
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
    }
    componentBulkLoader.end();
    return component;
}

Example 8 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMInvertedIndex method createMergeOperation.

@Override
protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException {
    ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(getLsmHarness(), opCtx);
    IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(opCtx);
    return new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
}

Example 9 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class LSMInvertedIndex method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
    // Create an inverted index instance to be bulk loaded.
    LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, flushOp.getTarget(), flushOp.getDeletedKeysBTreeTarget(), flushOp.getBloomFilterTarget(), true);
    // Create a scan cursor on the BTree underlying the in-memory inverted index.
    LSMInvertedIndexMemoryComponent flushingComponent = (LSMInvertedIndexMemoryComponent) flushOp.getFlushingComponent();
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    // Search the deleted keys BTree to calculate the number of elements for BloomFilter
    IIndexAccessor deletedKeysBTreeAccessor = flushingComponent.getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor btreeCountingCursor = ((BTreeAccessor) deletedKeysBTreeAccessor).createCountingSearchCursor();
    deletedKeysBTreeAccessor.search(btreeCountingCursor, nullPred);
    long numBTreeTuples = 0L;
    try {
        while (btreeCountingCursor.hasNext()) {
            btreeCountingCursor.next();
            ITupleReference countTuple = btreeCountingCursor.getTuple();
            numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
        }
    } finally {
        btreeCountingCursor.close();
    }
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
    // Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
    IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor(false);
    deletedKeysBTreeAccessor.search(deletedKeysScanCursor, nullPred);
    try {
        while (deletedKeysScanCursor.hasNext()) {
            deletedKeysScanCursor.next();
            ((LSMInvertedIndexDiskComponentBulkLoader) componentBulkLoader).delete(deletedKeysScanCursor.getTuple());
        }
    } finally {
        deletedKeysScanCursor.close();
    }
    // Scan the in-memory inverted index
    InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor();
    IIndexCursor scanCursor = memBTreeAccessor.createSearchCursor(false);
    memBTreeAccessor.search(scanCursor, nullPred);
    // Bulk load the disk inverted index from the in-memory inverted index.
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            componentBulkLoader.add(scanCursor.getTuple());
        }
    } finally {
        scanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
        filterManager.writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
    }
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}

Example 10 with IIndexCursor

use of org.apache.hyracks.storage.common.IIndexCursor in project asterixdb by apache.

the class OnDiskInvertedIndex method validate.

@Override
public void validate() throws HyracksDataException {
    btree.validate();
    // Scan the btree and validate the order of elements in each inverted-list.
    IIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
    MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
    RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
    int[] fieldPermutation = new int[tokenTypeTraits.length];
    for (int i = 0; i < tokenTypeTraits.length; i++) {
        fieldPermutation[i] = i;
    }
    PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
    IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
    MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
    try {
        // Search key for finding an inverted-list in the actual index.
        ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
        ArrayTupleReference prevTuple = new ArrayTupleReference();
        btreeAccessor.search(btreeCursor, rangePred);
        while (btreeCursor.hasNext()) {
            btreeCursor.next();
            tokenTuple.reset(btreeCursor.getTuple());
            // Validate inverted list by checking that the elements are totally ordered.
            invIndexAccessor.openInvertedListCursor(invListCursor, tokenTuple);
            invListCursor.pinPages();
            try {
                if (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Initialize prev tuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
                while (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Compare with previous element.
                    if (invListCmp.compare(invListElement, prevTuple) <= 0) {
                        throw new HyracksDataException("Index validation failed.");
                    }
                    // Set new prevTuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
            } finally {
                invListCursor.unpinPages();
            }
        }
    } finally {
        btreeCursor.close();
    }
}