Examples with MultiComparator org.apache.hyracks.storage.common.MultiComparator used on opensource projects

Example 6 with MultiComparator

use of org.apache.hyracks.storage.common.MultiComparator in project asterixdb by apache.

the class LSMInvertedIndexRangeSearchCursor method open.

@Override
public void open(ICursorInitialState initState, ISearchPredicate searchPred) throws HyracksDataException {
    LSMInvertedIndexRangeSearchCursorInitialState lsmInitState = (LSMInvertedIndexRangeSearchCursorInitialState) initState;
    cmp = lsmInitState.getOriginalKeyComparator();
    int numComponents = lsmInitState.getNumComponents();
    rangeCursors = new IIndexCursor[numComponents];
    for (int i = 0; i < numComponents; i++) {
        IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) lsmInitState.getIndexAccessors().get(i);
        rangeCursors[i] = invIndexAccessor.createRangeSearchCursor();
        invIndexAccessor.rangeSearch(rangeCursors[i], lsmInitState.getSearchPredicate());
    }
    lsmHarness = lsmInitState.getLSMHarness();
    operationalComponents = lsmInitState.getOperationalComponents();
    includeMutableComponent = lsmInitState.getIncludeMemComponent();
    // For searching the deleted-keys BTrees.
    this.keysOnlyTuple = lsmInitState.getKeysOnlyTuple();
    deletedKeysBTreeAccessors = lsmInitState.getDeletedKeysBTreeAccessors();
    if (!deletedKeysBTreeAccessors.isEmpty()) {
        deletedKeysBTreeCursors = new IIndexCursor[deletedKeysBTreeAccessors.size()];
        for (int i = 0; i < operationalComponents.size(); i++) {
            ILSMComponent component = operationalComponents.get(i);
            if (component.getType() == LSMComponentType.MEMORY) {
                // No need for a bloom filter for the in-memory BTree.
                deletedKeysBTreeCursors[i] = deletedKeysBTreeAccessors.get(i).createSearchCursor(false);
            } else {
                deletedKeysBTreeCursors[i] = new BloomFilterAwareBTreePointSearchCursor((IBTreeLeafFrame) lsmInitState.getgetDeletedKeysBTreeLeafFrameFactory().createFrame(), false, ((LSMInvertedIndexDiskComponent) operationalComponents.get(i)).getBloomFilter());
            }
        }
    }
    MultiComparator keyCmp = lsmInitState.getKeyComparator();
    keySearchPred = new RangePredicate(keysOnlyTuple, keysOnlyTuple, true, true, keyCmp, keyCmp);
    setPriorityQueueComparator();
    initPriorityQueue();
}

Example 7 with MultiComparator

use of org.apache.hyracks.storage.common.MultiComparator in project asterixdb by apache.

the class OnDiskInvertedIndex method validate.

@Override
public void validate() throws HyracksDataException {
    btree.validate();
    // Scan the btree and validate the order of elements in each inverted-list.
    IIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
    MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
    RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
    int[] fieldPermutation = new int[tokenTypeTraits.length];
    for (int i = 0; i < tokenTypeTraits.length; i++) {
        fieldPermutation[i] = i;
    }
    PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
    IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
    MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
    try {
        // Search key for finding an inverted-list in the actual index.
        ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
        ArrayTupleReference prevTuple = new ArrayTupleReference();
        btreeAccessor.search(btreeCursor, rangePred);
        while (btreeCursor.hasNext()) {
            btreeCursor.next();
            tokenTuple.reset(btreeCursor.getTuple());
            // Validate inverted list by checking that the elements are totally ordered.
            invIndexAccessor.openInvertedListCursor(invListCursor, tokenTuple);
            invListCursor.pinPages();
            try {
                if (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Initialize prev tuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
                while (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Compare with previous element.
                    if (invListCmp.compare(invListElement, prevTuple) <= 0) {
                        throw new HyracksDataException("Index validation failed.");
                    }
                    // Set new prevTuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
            } finally {
                invListCursor.unpinPages();
            }
        }
    } finally {
        btreeCursor.close();
    }
}

Example 8 with MultiComparator

use of org.apache.hyracks.storage.common.MultiComparator in project asterixdb by apache.

the class BTreeNSMInteriorFrame method getChildPageId.

@Override
public int getChildPageId(RangePredicate pred) throws HyracksDataException {
    // Trivial case where there is only a child pointer (and no key).
    if (buf.getInt(Constants.TUPLE_COUNT_OFFSET) == 0) {
        return buf.getInt(RIGHT_LEAF_OFFSET);
    }
    // Trivial cases where no low key or high key was given (e.g. during an
    // index scan).
    ITupleReference tuple = null;
    FindTupleMode fsm = null;
    // The target comparator may be on a prefix of the BTree key fields.
    MultiComparator targetCmp = pred.getLowKeyComparator();
    tuple = pred.getLowKey();
    if (tuple == null) {
        return getLeftmostChildPageId();
    }
    if (pred.isLowKeyInclusive()) {
        fsm = FindTupleMode.INCLUSIVE;
    } else {
        fsm = FindTupleMode.EXCLUSIVE;
    }
    // Search for a matching key.
    int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, targetCmp, fsm, FindTupleNoExactMatchPolicy.HIGHER_KEY);
    int slotOff = slotManager.getSlotOff(tupleIndex);
    // Follow the rightmost (greatest) child pointer.
    if (tupleIndex == slotManager.getGreatestKeyIndicator()) {
        return buf.getInt(RIGHT_LEAF_OFFSET);
    }
    // Deal with prefix searches.
    // slotManager.findTupleIndex() will return an arbitrary tuple matching
    // the given field prefix (according to the target comparator).
    // To make sure we traverse the right path, we must find the
    // leftmost or rightmost tuple that matches the prefix.
    int origTupleOff = slotManager.getTupleOff(slotOff);
    cmpFrameTuple.resetByTupleOffset(buf.array(), origTupleOff);
    int cmpTupleOff = origTupleOff;
    // The answer set begins with the lowest key matching the prefix.
    // We must follow the child pointer of the lowest (leftmost) key
    // matching the given prefix.
    int maxSlotOff = buf.capacity();
    slotOff += slotManager.getSlotSize();
    while (slotOff < maxSlotOff) {
        cmpTupleOff = slotManager.getTupleOff(slotOff);
        frameTuple.resetByTupleOffset(buf.array(), cmpTupleOff);
        if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0) {
            break;
        }
        slotOff += slotManager.getSlotSize();
    }
    slotOff -= slotManager.getSlotSize();
    frameTuple.resetByTupleOffset(buf.array(), slotManager.getTupleOff(slotOff));
    int childPageOff = getLeftChildPageOff(frameTuple);
    return buf.getInt(childPageOff);
}

Example 9 with MultiComparator

use of org.apache.hyracks.storage.common.MultiComparator in project asterixdb by apache.

the class BTree method printTree.

@SuppressWarnings("rawtypes")
public String printTree(IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, ISerializerDeserializer[] keySerdes) throws Exception {
    MultiComparator cmp = MultiComparator.create(cmpFactories);
    byte treeHeight = getTreeHeight(leafFrame);
    StringBuilder strBuilder = new StringBuilder();
    printTree(rootPage, null, false, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder, cmp);
    return strBuilder.toString();
}

Example 10 with MultiComparator

use of org.apache.hyracks.storage.common.MultiComparator in project asterixdb by apache.

the class LSMInvertedIndexTestUtils method compareActualAndExpectedIndexes.

/**
     * Compares actual and expected indexes by comparing their inverted-lists one by one. Exercises the openInvertedListCursor() method of the inverted-index accessor.
     */
@SuppressWarnings("unchecked")
public static void compareActualAndExpectedIndexes(LSMInvertedIndexTestContext testCtx) throws HyracksDataException {
    IInvertedIndex invIndex = (IInvertedIndex) testCtx.getIndex();
    ISerializerDeserializer[] fieldSerdes = testCtx.getFieldSerdes();
    MultiComparator invListCmp = MultiComparator.create(invIndex.getInvListCmpFactories());
    IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) testCtx.getIndexAccessor();
    int tokenFieldCount = invIndex.getTokenTypeTraits().length;
    int invListFieldCount = invIndex.getInvListTypeTraits().length;
    // All tokens that were inserted into the indexes.
    Iterator<Comparable> tokensIter = testCtx.getAllTokens().iterator();
    // Search key for finding an inverted-list in the actual index.
    ArrayTupleBuilder searchKeyBuilder = new ArrayTupleBuilder(tokenFieldCount);
    ArrayTupleReference searchKey = new ArrayTupleReference();
    // Cursor over inverted list from actual index.
    IInvertedListCursor actualInvListCursor = invIndexAccessor.createInvertedListCursor();
    // Helpers for generating a serialized inverted-list element from a CheckTuple from the expected index.
    ArrayTupleBuilder expectedBuilder = new ArrayTupleBuilder(fieldSerdes.length);
    // Includes the token fields.
    ArrayTupleReference completeExpectedTuple = new ArrayTupleReference();
    // Field permutation and permuting tuple reference to strip away token fields from completeExpectedTuple.
    int[] fieldPermutation = new int[invListFieldCount];
    for (int i = 0; i < fieldPermutation.length; i++) {
        fieldPermutation[i] = tokenFieldCount + i;
    }
    PermutingTupleReference expectedTuple = new PermutingTupleReference(fieldPermutation);
    // Iterate over all tokens. Find the inverted-lists in actual and expected indexes. Compare the inverted lists,
    while (tokensIter.hasNext()) {
        Comparable token = tokensIter.next();
        // Position inverted-list iterator on expected index.
        CheckTuple checkLowKey = new CheckTuple(tokenFieldCount, tokenFieldCount);
        checkLowKey.appendField(token);
        CheckTuple checkHighKey = new CheckTuple(tokenFieldCount, tokenFieldCount);
        checkHighKey.appendField(token);
        SortedSet<CheckTuple> expectedInvList = OrderedIndexTestUtils.getPrefixExpectedSubset(testCtx.getCheckTuples(), checkLowKey, checkHighKey);
        Iterator<CheckTuple> expectedInvListIter = expectedInvList.iterator();
        // Position inverted-list cursor in actual index.
        OrderedIndexTestUtils.createTupleFromCheckTuple(checkLowKey, searchKeyBuilder, searchKey, fieldSerdes);
        invIndexAccessor.openInvertedListCursor(actualInvListCursor, searchKey);
        if (actualInvListCursor.size() != expectedInvList.size()) {
            fail("Actual and expected inverted lists for token '" + token.toString() + "' have different sizes. Actual size: " + actualInvListCursor.size() + ". Expected size: " + expectedInvList.size() + ".");
        }
        // Compare inverted-list elements.
        int count = 0;
        actualInvListCursor.pinPages();
        try {
            while (actualInvListCursor.hasNext() && expectedInvListIter.hasNext()) {
                actualInvListCursor.next();
                ITupleReference actual = actualInvListCursor.getTuple();
                CheckTuple expected = expectedInvListIter.next();
                OrderedIndexTestUtils.createTupleFromCheckTuple(expected, expectedBuilder, completeExpectedTuple, fieldSerdes);
                expectedTuple.reset(completeExpectedTuple);
                if (invListCmp.compare(actual, expectedTuple) != 0) {
                    fail("Inverted lists of token '" + token + "' differ at position " + count + ".");
                }
                count++;
            }
        } finally {
            actualInvListCursor.unpinPages();
        }
    }
}