Examples with IIndexAccessor org.apache.hyracks.storage.common.IIndexAccessor used on opensource projects

Example 11 with IIndexAccessor

use of org.apache.hyracks.storage.common.IIndexAccessor in project asterixdb by apache.

the class LSMInvertedIndex method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
    // Create an inverted index instance to be bulk loaded.
    LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, flushOp.getTarget(), flushOp.getDeletedKeysBTreeTarget(), flushOp.getBloomFilterTarget(), true);
    // Create a scan cursor on the BTree underlying the in-memory inverted index.
    LSMInvertedIndexMemoryComponent flushingComponent = (LSMInvertedIndexMemoryComponent) flushOp.getFlushingComponent();
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    // Search the deleted keys BTree to calculate the number of elements for BloomFilter
    IIndexAccessor deletedKeysBTreeAccessor = flushingComponent.getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor btreeCountingCursor = ((BTreeAccessor) deletedKeysBTreeAccessor).createCountingSearchCursor();
    deletedKeysBTreeAccessor.search(btreeCountingCursor, nullPred);
    long numBTreeTuples = 0L;
    try {
        while (btreeCountingCursor.hasNext()) {
            btreeCountingCursor.next();
            ITupleReference countTuple = btreeCountingCursor.getTuple();
            numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
        }
    } finally {
        btreeCountingCursor.close();
    }
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
    // Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
    IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor(false);
    deletedKeysBTreeAccessor.search(deletedKeysScanCursor, nullPred);
    try {
        while (deletedKeysScanCursor.hasNext()) {
            deletedKeysScanCursor.next();
            ((LSMInvertedIndexDiskComponentBulkLoader) componentBulkLoader).delete(deletedKeysScanCursor.getTuple());
        }
    } finally {
        deletedKeysScanCursor.close();
    }
    // Scan the in-memory inverted index
    InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor();
    IIndexCursor scanCursor = memBTreeAccessor.createSearchCursor(false);
    memBTreeAccessor.search(scanCursor, nullPred);
    // Bulk load the disk inverted index from the in-memory inverted index.
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            componentBulkLoader.add(scanCursor.getTuple());
        }
    } finally {
        scanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
        filterManager.writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
    }
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}

Example 12 with IIndexAccessor

use of org.apache.hyracks.storage.common.IIndexAccessor in project asterixdb by apache.

the class OnDiskInvertedIndex method validate.

@Override
public void validate() throws HyracksDataException {
    btree.validate();
    // Scan the btree and validate the order of elements in each inverted-list.
    IIndexAccessor btreeAccessor = btree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor btreeCursor = btreeAccessor.createSearchCursor(false);
    MultiComparator btreeCmp = MultiComparator.create(btree.getComparatorFactories());
    RangePredicate rangePred = new RangePredicate(null, null, true, true, btreeCmp, btreeCmp);
    int[] fieldPermutation = new int[tokenTypeTraits.length];
    for (int i = 0; i < tokenTypeTraits.length; i++) {
        fieldPermutation[i] = i;
    }
    PermutingTupleReference tokenTuple = new PermutingTupleReference(fieldPermutation);
    IInvertedIndexAccessor invIndexAccessor = (IInvertedIndexAccessor) createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IInvertedListCursor invListCursor = invIndexAccessor.createInvertedListCursor();
    MultiComparator invListCmp = MultiComparator.create(invListCmpFactories);
    try {
        // Search key for finding an inverted-list in the actual index.
        ArrayTupleBuilder prevBuilder = new ArrayTupleBuilder(invListTypeTraits.length);
        ArrayTupleReference prevTuple = new ArrayTupleReference();
        btreeAccessor.search(btreeCursor, rangePred);
        while (btreeCursor.hasNext()) {
            btreeCursor.next();
            tokenTuple.reset(btreeCursor.getTuple());
            // Validate inverted list by checking that the elements are totally ordered.
            invIndexAccessor.openInvertedListCursor(invListCursor, tokenTuple);
            invListCursor.pinPages();
            try {
                if (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Initialize prev tuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
                while (invListCursor.hasNext()) {
                    invListCursor.next();
                    ITupleReference invListElement = invListCursor.getTuple();
                    // Compare with previous element.
                    if (invListCmp.compare(invListElement, prevTuple) <= 0) {
                        throw new HyracksDataException("Index validation failed.");
                    }
                    // Set new prevTuple.
                    TupleUtils.copyTuple(prevBuilder, invListElement, invListElement.getFieldCount());
                    prevTuple.reset(prevBuilder.getFieldEndOffsets(), prevBuilder.getByteArray());
                }
            } finally {
                invListCursor.unpinPages();
            }
        }
    } finally {
        btreeCursor.close();
    }
}

Example 13 with IIndexAccessor

use of org.apache.hyracks.storage.common.IIndexAccessor in project asterixdb by apache.

the class LSMBuddyBTreeMergeCursor method open.

@Override
public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
    LSMBTreeWithBuddyCursorInitialState lsmInitialState = (LSMBTreeWithBuddyCursorInitialState) initialState;
    cmp = lsmInitialState.getBuddyBTreeCmp();
    operationalComponents = lsmInitialState.getOperationalComponents();
    // We intentionally set the lsmHarness to null so that we don't call
    // lsmHarness.endSearch() because we already do that when we merge
    // actual index.
    lsmHarness = null;
    int numBTrees = operationalComponents.size();
    rangeCursors = new IIndexCursor[numBTrees];
    RangePredicate btreePredicate = new RangePredicate(null, null, true, true, cmp, cmp);
    IIndexAccessor[] btreeAccessors = new ITreeIndexAccessor[numBTrees];
    for (int i = 0; i < numBTrees; i++) {
        ILSMComponent component = operationalComponents.get(i);
        IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) lsmInitialState.getBuddyBTreeLeafFrameFactory().createFrame();
        rangeCursors[i] = new BTreeRangeSearchCursor(leafFrame, false);
        BTree buddyBtree = ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree();
        btreeAccessors[i] = buddyBtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        btreeAccessors[i].search(rangeCursors[i], btreePredicate);
    }
    setPriorityQueueComparator();
    initPriorityQueue();
}

Example 14 with IIndexAccessor

use of org.apache.hyracks.storage.common.IIndexAccessor in project asterixdb by apache.

the class LSMBTree method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
    LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent();
    IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        IIndexCursor countingCursor = ((BTreeAccessor) accessor).createCountingSearchCursor();
        accessor.search(countingCursor, nullPred);
        try {
            while (countingCursor.hasNext()) {
                countingCursor.next();
                ITupleReference countTuple = countingCursor.getTuple();
                numElements = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
            }
        } finally {
            countingCursor.close();
        }
    }
    LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
    IIndexCursor scanCursor = accessor.createSearchCursor(false);
    accessor.search(scanCursor, nullPred);
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            componentBulkLoader.add(scanCursor.getTuple());
        }
    } finally {
        scanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getBTree());
    }
    // Write metadata from memory component to disk
    // Q. what about the merge operation? how do we resolve conflicts
    // A. Through providing an appropriate ILSMIOOperationCallback
    // Must not reset the metadata before the flush is completed
    // Use the copy of the metadata in the opContext
    // TODO This code should be in the callback and not in the index
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}

Example 15 with IIndexAccessor

use of org.apache.hyracks.storage.common.IIndexAccessor in project asterixdb by apache.

the class FramewriterTest method mockIndexes.

public ITreeIndex[] mockIndexes() throws HyracksDataException {
    IIndexAccessor[] indexAccessors = mockIndexAccessors();
    ITreeIndex[] indexes = new ITreeIndex[indexAccessors.length * 2];
    int j = 0;
    for (int i = 0; i < indexAccessors.length; i++) {
        indexes[j] = Mockito.mock(ITreeIndex.class);
        Mockito.when(indexes[j].createAccessor(Mockito.any(), Mockito.any())).thenReturn(indexAccessors[i]);
        j++;
        indexes[j] = Mockito.mock(ITreeIndex.class);
        Mockito.when(indexes[j].createAccessor(Mockito.any(), Mockito.any())).thenThrow(new HyracksDataException("failed to create accessor"));
        j++;
    }
    return indexes;
}