Examples with InvertedIndexSearchPredicate org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate used on opensource projects

Example 1 with InvertedIndexSearchPredicate

use of org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate in project asterixdb by apache.

the class LSMInvertedIndexSearchOperatorNodePushable method resetSearchPredicate.

@Override
protected void resetSearchPredicate(int tupleIndex) {
    frameTuple.reset(accessor, tupleIndex);
    InvertedIndexSearchPredicate invIndexSearchPred = (InvertedIndexSearchPredicate) searchPred;
    invIndexSearchPred.setQueryTuple(frameTuple);
    invIndexSearchPred.setQueryFieldIndex(queryFieldIndex);
    invIndexSearchPred.setIsFullTextSearchQuery(isFullTextSearchQuery);
    if (minFilterKey != null) {
        minFilterKey.reset(accessor, tupleIndex);
    }
    if (maxFilterKey != null) {
        maxFilterKey.reset(accessor, tupleIndex);
    }
}

Example 2 with InvertedIndexSearchPredicate

use of org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate in project asterixdb by apache.

the class LSMInvertedIndex method createCursorInitialState.

private ICursorInitialState createCursorInitialState(ISearchPredicate pred, IIndexOperationContext ictx, boolean includeMutableComponent, ArrayList<IIndexAccessor> indexAccessors, ArrayList<IIndexAccessor> deletedKeysBTreeAccessors, List<ILSMComponent> operationalComponents) {
    ICursorInitialState initState;
    PermutingTupleReference keysOnlyTuple = createKeysOnlyTupleReference();
    MultiComparator keyCmp = MultiComparator.create(invListCmpFactories);
    // Distinguish between regular searches and range searches (mostly used in merges).
    if (pred instanceof InvertedIndexSearchPredicate) {
        initState = new LSMInvertedIndexSearchCursorInitialState(keyCmp, keysOnlyTuple, indexAccessors, deletedKeysBTreeAccessors, ((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get())).getDeletedKeysBTree().getLeafFrameFactory(), ictx, includeMutableComponent, getLsmHarness(), operationalComponents);
    } else {
        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
        InMemoryInvertedIndex memInvIndex = (InMemoryInvertedIndex) mutableComponent.getInvIndex();
        MultiComparator tokensAndKeysCmp = MultiComparator.create(memInvIndex.getBTree().getComparatorFactories());
        initState = new LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp, keysOnlyTuple, ((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get())).getDeletedKeysBTree().getLeafFrameFactory(), includeMutableComponent, getLsmHarness(), indexAccessors, deletedKeysBTreeAccessors, pred, operationalComponents);
    }
    return initState;
}

Example 3 with InvertedIndexSearchPredicate

use of org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate in project asterixdb by apache.

the class LSMInvertedIndexTestUtils method testIndexSearch.

public static void testIndexSearch(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen, Random rnd, int numDocQueries, int numRandomQueries, IInvertedIndexSearchModifier searchModifier, int[] scanCountArray) throws IOException, HyracksDataException {
    IInvertedIndex invIndex = testCtx.invIndex;
    IInvertedIndexAccessor accessor = (IInvertedIndexAccessor) invIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IBinaryTokenizer tokenizer = testCtx.getTokenizerFactory().createTokenizer();
    InvertedIndexSearchPredicate searchPred = new InvertedIndexSearchPredicate(tokenizer, searchModifier);
    List<ITupleReference> documentCorpus = testCtx.getDocumentCorpus();
    // Project away the primary-key field.
    int[] fieldPermutation = new int[] { 0 };
    PermutingTupleReference searchDocument = new PermutingTupleReference(fieldPermutation);
    int numQueries = numDocQueries + numRandomQueries;
    for (int i = 0; i < numQueries; i++) {
        // If number of documents in the corpus is less than numDocQueries, then replace the remaining ones with random queries.
        if (i >= numDocQueries || i >= documentCorpus.size()) {
            // Generate a random query.
            ITupleReference randomQuery = tupleGen.next();
            searchDocument.reset(randomQuery);
        } else {
            // Pick a random document from the corpus to use as the search query.
            int queryIndex = Math.abs(rnd.nextInt() % documentCorpus.size());
            searchDocument.reset(documentCorpus.get(queryIndex));
        }
        // Set query tuple in search predicate.
        searchPred.setQueryTuple(searchDocument);
        searchPred.setQueryFieldIndex(0);
        IIndexCursor resultCursor = accessor.createSearchCursor(false);
        boolean panic = false;
        try {
            accessor.search(resultCursor, searchPred);
        } catch (HyracksDataException e) {
            // ignore panic queries.
            if (e.getErrorCode() == ErrorCode.OCCURRENCE_THRESHOLD_PANIC_EXCEPTION) {
                panic = true;
            } else {
                throw e;
            }
        }
        try {
            if (!panic) {
                // Consume cursor and deserialize results so we can sort them. Some search cursors may not deliver the result sorted (e.g., LSM search cursor).
                ArrayList<Integer> actualResults = new ArrayList<>();
                try {
                    while (resultCursor.hasNext()) {
                        resultCursor.next();
                        ITupleReference resultTuple = resultCursor.getTuple();
                        int actual = IntegerPointable.getInteger(resultTuple.getFieldData(0), resultTuple.getFieldStart(0));
                        actualResults.add(Integer.valueOf(actual));
                    }
                } catch (HyracksDataException e) {
                    if (e.getErrorCode() == ErrorCode.OCCURRENCE_THRESHOLD_PANIC_EXCEPTION) {
                        // Ignore panic queries.
                        continue;
                    } else {
                        throw e;
                    }
                }
                Collections.sort(actualResults);
                // Get expected results.
                List<Integer> expectedResults = new ArrayList<>();
                LSMInvertedIndexTestUtils.getExpectedResults(scanCountArray, testCtx.getCheckTuples(), searchDocument, tokenizer, testCtx.getFieldSerdes()[0], searchModifier, expectedResults, testCtx.getInvertedIndexType());
                Iterator<Integer> expectedIter = expectedResults.iterator();
                Iterator<Integer> actualIter = actualResults.iterator();
                while (expectedIter.hasNext() && actualIter.hasNext()) {
                    int expected = expectedIter.next();
                    int actual = actualIter.next();
                    if (actual != expected) {
                        fail("Query results do not match. Encountered: " + actual + ". Expected: " + expected + "");
                    }
                }
                if (expectedIter.hasNext()) {
                    fail("Query results do not match. Actual results missing.");
                }
                if (actualIter.hasNext()) {
                    fail("Query results do not match. Actual contains too many results.");
                }
            }
        } finally {
            resultCursor.close();
        }
    }
}

Example 4 with InvertedIndexSearchPredicate

use of org.apache.hyracks.storage.am.lsm.invertedindex.search.InvertedIndexSearchPredicate in project asterixdb by apache.

the class LSMInvertedIndexTestWorker method performOp.

@Override
public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
    LSMInvertedIndexAccessor accessor = (LSMInvertedIndexAccessor) indexAccessor;
    IIndexCursor searchCursor = accessor.createSearchCursor(false);
    IIndexCursor rangeSearchCursor = accessor.createRangeSearchCursor();
    RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
    IBinaryTokenizerFactory tokenizerFactory = invIndex.getTokenizerFactory();
    int searchModifierIndex = Math.abs(rnd.nextInt()) % TEST_SEARCH_MODIFIERS.length;
    InvertedIndexSearchPredicate searchPred = new InvertedIndexSearchPredicate(tokenizerFactory.createTokenizer(), TEST_SEARCH_MODIFIERS[searchModifierIndex]);
    switch(op) {
        case INSERT:
            {
                insert(accessor, tuple);
                break;
            }
        case DELETE:
            {
                // Randomly pick a document from the corpus to delete.
                if (!documentCorpus.isEmpty()) {
                    int docIndex = Math.abs(rnd.nextInt()) % documentCorpus.size();
                    ITupleReference deleteTuple = documentCorpus.get(docIndex);
                    accessor.delete(deleteTuple);
                    // Swap tupleIndex with last element.
                    documentCorpus.set(docIndex, documentCorpus.get(documentCorpus.size() - 1));
                    documentCorpus.remove(documentCorpus.size() - 1);
                } else {
                    // No existing documents to delete, treat this case as an insert.
                    insert(accessor, tuple);
                }
                break;
            }
        case POINT_SEARCH:
            {
                searchCursor.reset();
                searchPred.setQueryTuple(tuple);
                searchPred.setQueryFieldIndex(0);
                try {
                    accessor.search(searchCursor, searchPred);
                    consumeCursorTuples(searchCursor);
                } catch (HyracksDataException e) {
                    // Ignore.
                    if (e.getErrorCode() != ErrorCode.OCCURRENCE_THRESHOLD_PANIC_EXCEPTION) {
                        throw e;
                    }
                }
                break;
            }
        case SCAN:
            {
                rangeSearchCursor.reset();
                accessor.rangeSearch(rangeSearchCursor, rangePred);
                consumeCursorTuples(rangeSearchCursor);
                break;
            }
        case MERGE:
            {
                accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), invIndex.getImmutableComponents());
                break;
            }
        default:
            throw new HyracksDataException("Op " + op.toString() + " not supported.");
    }
}