Examples with RangePredicate org.apache.hyracks.storage.am.btree.impls.RangePredicate used on opensource projects

Example 11 with RangePredicate

use of org.apache.hyracks.storage.am.btree.impls.RangePredicate in project asterixdb by apache.

the class OnDiskInvertedIndex method validate.

@Override
public void validate() throws HyracksDataException {
    btree.validate();
    // Scan the btree and validate the order of elements in each inverted-list.
    IIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
    MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
    RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
    int[] fieldPermutation = new int[tokenTypeTraits.length];
    for (int i = 0; i < tokenTypeTraits.length; i++) {
        fieldPermutation[i] = i;
    }
    PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
    IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
    MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
    try {
        // Search key for finding an inverted-list in the actual index.
        ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
        ArrayTupleReference prevTuple = new ArrayTupleReference();
        btreeAccessor.search(btreeCursor, rangePred);
        while (btreeCursor.hasNext()) {
            btreeCursor.next();
            tokenTuple.reset(btreeCursor.getTuple());
            // Validate inverted list by checking that the elements are totally ordered.
            invIndexAccessor.openInvertedListCursor(invListCursor, tokenTuple);
            invListCursor.pinPages();
            try {
                if (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Initialize prev tuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
                while (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Compare with previous element.
                    if (invListCmp.compare(invListElement, prevTuple) <= 0) {
                        throw new HyracksDataException("Index validation failed.");
                    }
                    // Set new prevTuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
            } finally {
                invListCursor.unpinPages();
            }
        }
    } finally {
        btreeCursor.close();
    }
}

Example 12 with RangePredicate

use of org.apache.hyracks.storage.am.btree.impls.RangePredicate in project asterixdb by apache.

the class LSMBTree method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
    LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent();
    IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        IIndexCursor countingCursor = ((BTreeAccessor) accessor).createCountingSearchCursor();
        accessor.search(countingCursor, nullPred);
        try {
            while (countingCursor.hasNext()) {
                countingCursor.next();
                ITupleReference countTuple = countingCursor.getTuple();
                numElements = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
            }
        } finally {
            countingCursor.close();
        }
    }
    LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
    IIndexCursor scanCursor = accessor.createSearchCursor(false);
    accessor.search(scanCursor, nullPred);
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            componentBulkLoader.add(scanCursor.getTuple());
        }
    } finally {
        scanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getBTree());
    }
    // Write metadata from memory component to disk
    // Q. what about the merge operation? how do we resolve conflicts
    // A. Through providing an appropriate ILSMIOOperationCallback
    // Must not reset the metadata before the flush is completed
    // Use the copy of the metadata in the opContext
    // TODO This code should be in the callback and not in the index
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}

Example 13 with RangePredicate

use of org.apache.hyracks.storage.am.btree.impls.RangePredicate in project asterixdb by apache.

the class LSMBTree method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rangePred);
    List<ILSMComponent> mergedComponents = mergeOp.getMergingComponents();
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        for (int i = 0; i < mergedComponents.size(); ++i) {
            numElements += ((LSMBTreeDiskComponent) mergedComponents.get(i)).getBloomFilter().getNumElements();
        }
    }
    LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (mergedComponent.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
    }
    componentBulkLoader.end();
    return mergedComponent;
}

Example 14 with RangePredicate

use of org.apache.hyracks.storage.am.btree.impls.RangePredicate in project asterixdb by apache.

the class LSMBTree method insert.

private boolean insert(ITupleReference tuple, LSMBTreeOpContext ctx) throws HyracksDataException {
    LSMBTreePointSearchCursor searchCursor = ctx.getInsertSearchCursor();
    IIndexCursor memCursor = ctx.getMemCursor();
    RangePredicate predicate = (RangePredicate) ctx.getSearchPredicate();
    predicate.setHighKey(tuple);
    predicate.setLowKey(tuple);
    if (needKeyDupCheck) {
        // first check the inmemory component
        ctx.getCurrentMutableBTreeAccessor().search(memCursor, predicate);
        try {
            if (memCursor.hasNext()) {
                memCursor.next();
                LSMBTreeTupleReference lsmbtreeTuple = (LSMBTreeTupleReference) memCursor.getTuple();
                if (!lsmbtreeTuple.isAntimatter()) {
                    throw HyracksDataException.create(ErrorCode.DUPLICATE_KEY);
                } else {
                    memCursor.close();
                    ctx.getCurrentMutableBTreeAccessor().upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
                    return true;
                }
            }
        } finally {
            memCursor.close();
        }
        // TODO: Can we just remove the above code that search the mutable
        // component and do it together with the search call below? i.e. instead
        // of passing false to the lsmHarness.search(), we pass true to include
        // the mutable component?
        // the key was not in the inmemory component, so check the disk
        // components
        // This is a hack to avoid searching the current active mutable component twice. It is critical to add it back once the search is over.
        ILSMComponent firstComponent = ctx.getComponentHolder().remove(0);
        search(ctx, searchCursor, predicate);
        try {
            if (searchCursor.hasNext()) {
                throw HyracksDataException.create(ErrorCode.DUPLICATE_KEY);
            }
        } finally {
            searchCursor.close();
            // Add the current active mutable component back
            ctx.getComponentHolder().add(0, firstComponent);
        }
    }
    ctx.getCurrentMutableBTreeAccessor().upsertIfConditionElseInsert(tuple, AntimatterAwareTupleAcceptor.INSTANCE);
    return true;
}

Example 15 with RangePredicate

use of org.apache.hyracks.storage.am.btree.impls.RangePredicate in project asterixdb by apache.

the class LSMInvertedIndexTestUtils method compareActualAndExpectedIndexesRangeSearch.

/**
     * Compares actual and expected indexes using the rangeSearch() method of the inverted-index accessor.
     */
public static void compareActualAndExpectedIndexesRangeSearch(LSMInvertedIndexTestContext testCtx) throws HyracksDataException {
    IInvertedIndex invIndex = (IInvertedIndex) testCtx.getIndex();
    int tokenFieldCount = invIndex.getTokenTypeTraits().length;
    int invListFieldCount = invIndex.getInvListTypeTraits().length;
    IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) invIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor invIndexCursor = invIndexAccessor.createRangeSearchCursor();
    MultiComparator tokenCmp = MultiComparator.create(invIndex.getTokenCmpFactories());
    IBinaryComparatorFactory[] tupleCmpFactories = new IBinaryComparatorFactory[tokenFieldCount + invListFieldCount];
    for (int i = 0; i < tokenFieldCount; i++) {
        tupleCmpFactories[i] = invIndex.getTokenCmpFactories()[i];
    }
    for (int i = 0; i < invListFieldCount; i++) {
        tupleCmpFactories[tokenFieldCount + i] = invIndex.getInvListCmpFactories()[i];
    }
    MultiComparator tupleCmp = MultiComparator.create(tupleCmpFactories);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, tokenCmp, tokenCmp);
    invIndexAccessor.rangeSearch(invIndexCursor, nullPred);
    // Helpers for generating a serialized inverted-list element from a CheckTuple from the expected index.
    ISerializerDeserializer[] fieldSerdes = testCtx.getFieldSerdes();
    ArrayTupleBuilder expectedBuilder = new ArrayTupleBuilder(fieldSerdes.length);
    ArrayTupleReference expectedTuple = new ArrayTupleReference();
    Iterator<CheckTuple> expectedIter = testCtx.getCheckTuples().iterator();
    // Compare index elements.
    try {
        while (invIndexCursor.hasNext() && expectedIter.hasNext()) {
            invIndexCursor.next();
            ITupleReference actualTuple = invIndexCursor.getTuple();
            CheckTuple expected = expectedIter.next();
            OrderedIndexTestUtils.createTupleFromCheckTuple(expected, expectedBuilder, expectedTuple, fieldSerdes);
            if (tupleCmp.compare(actualTuple, expectedTuple) != 0) {
                fail("Index entries differ for token '" + expected.getField(0) + "'.");
            }
        }
        if (expectedIter.hasNext()) {
            fail("Indexes do not match. Actual index is missing entries.");
        }
        if (invIndexCursor.hasNext()) {
            fail("Indexes do not match. Actual index contains too many entries.");
        }
    } finally {
        invIndexCursor.close();
    }
}