Examples with ILSMIndexAccessor org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor used on opensource projects

Example 1 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMRTreeWithAntiMatterTuples method createMergeOperation.

@Override
protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException {
    boolean returnDeletedTuples = false;
    if (mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
        returnDeletedTuples = true;
    }
    ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx, returnDeletedTuples);
    ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory);
    return new MergeOperation(accessor, mergeFileRefs.getInsertIndexFileReference(), callback, fileManager.getBaseDir(), mergingComponents, cursor);
}

Example 2 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMBTreeFilterMergeTestDriver method runTest.

@Override
protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
    OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType, true);
    ctx.getIndex().create();
    ctx.getIndex().activate();
    // to determine which field types to generate.
    if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, true, getRandom());
    } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, true, getRandom());
    }
    int maxTreesToMerge = AccessMethodTestsConfig.LSM_BTREE_MAX_TREES_TO_MERGE;
    ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
    IBinaryComparator comp = ctx.getComparatorFactories()[0].createBinaryComparator();
    for (int i = 0; i < maxTreesToMerge; i++) {
        int flushed = 0;
        for (; flushed < i; flushed++) {
            Pair<ITupleReference, ITupleReference> minMax = null;
            if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
                minMax = orderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, true, getRandom());
            } else {
                minMax = orderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, true, getRandom());
            }
            if (minMax != null) {
                ILSMComponentFilter f = ((LSMBTree) ctx.getIndex()).getCurrentMemoryComponent().getLSMComponentFilter();
                Pair<ITupleReference, ITupleReference> obsMinMax = filterToMinMax(f);
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getLeft(), minMax.getLeft(), comp));
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getRight(), minMax.getRight(), comp));
            }
            StubIOOperationCallback stub = new StubIOOperationCallback();
            BlockingIOOperationCallbackWrapper waiter = new BlockingIOOperationCallbackWrapper(stub);
            accessor.scheduleFlush(waiter);
            waiter.waitForIO();
            if (minMax != null) {
                Pair<ITupleReference, ITupleReference> obsMinMax = filterToMinMax(stub.getLastNewComponent().getLSMComponentFilter());
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getLeft(), minMax.getLeft(), comp));
                Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getRight(), minMax.getRight(), comp));
            }
        }
        List<ILSMDiskComponent> flushedComponents = ((LSMBTree) ctx.getIndex()).getImmutableComponents();
        MutablePair<ITupleReference, ITupleReference> expectedMergeMinMax = null;
        for (ILSMDiskComponent f : flushedComponents) {
            Pair<ITupleReference, ITupleReference> componentMinMax = filterToMinMax(f.getLSMComponentFilter());
            if (expectedMergeMinMax == null) {
                expectedMergeMinMax = MutablePair.of(componentMinMax.getLeft(), componentMinMax.getRight());
            }
            if (TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getLeft(), componentMinMax.getLeft(), comp) > 0) {
                expectedMergeMinMax.setLeft(componentMinMax.getLeft());
            }
            if (TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getRight(), componentMinMax.getRight(), comp) < 0) {
                expectedMergeMinMax.setRight(componentMinMax.getRight());
            }
        }
        accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), ((LSMBTree) ctx.getIndex()).getImmutableComponents());
        flushedComponents = ((LSMBTree) ctx.getIndex()).getImmutableComponents();
        Pair<ITupleReference, ITupleReference> mergedMinMax = filterToMinMax(flushedComponents.get(0).getLSMComponentFilter());
        Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getLeft(), mergedMinMax.getLeft(), comp));
        Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getRight(), mergedMinMax.getRight(), comp));
        orderedIndexTestUtils.checkPointSearches(ctx);
        orderedIndexTestUtils.checkScan(ctx);
        orderedIndexTestUtils.checkDiskOrderScan(ctx);
        orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
        if (prefixLowKey != null && prefixHighKey != null) {
            orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
        }
    }
    ctx.getIndex().deactivate();
    ctx.getIndex().destroy();
}

Example 3 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class MetadataNode method insertTupleIntoIndex.

private void insertTupleIntoIndex(JobId jobId, IMetadataIndex metadataIndex, ITupleReference tuple) throws ACIDException, HyracksDataException {
    long resourceID = metadataIndex.getResourceId();
    String resourceName = metadataIndex.getFile().getRelativePath();
    ILSMIndex lsmIndex = (ILSMIndex) datasetLifecycleManager.get(resourceName);
    try {
        datasetLifecycleManager.open(resourceName);
        // prepare a Callback for logging
        IModificationOperationCallback modCallback = createIndexModificationCallback(jobId, resourceID, metadataIndex, lsmIndex, Operation.INSERT);
        ILSMIndexAccessor indexAccessor = lsmIndex.createAccessor(modCallback, NoOpOperationCallback.INSTANCE);
        ITransactionContext txnCtx = transactionSubsystem.getTransactionManager().getTransactionContext(jobId, false);
        txnCtx.setWriteTxn(true);
        txnCtx.registerIndexAndCallback(resourceID, lsmIndex, (AbstractOperationCallback) modCallback, metadataIndex.isPrimaryIndex());
        LSMIndexUtil.checkAndSetFirstLSN((AbstractLSMIndex) lsmIndex, transactionSubsystem.getLogManager());
        // TODO: fix exceptions once new BTree exception model is in hyracks.
        indexAccessor.forceInsert(tuple);
        //by the job commit log event
        if (!((TransactionContext) txnCtx).getPrimaryIndexOpTracker().equals(lsmIndex.getOperationTracker())) {
            lsmIndex.getOperationTracker().completeOperation(lsmIndex, LSMOperationType.FORCE_MODIFICATION, null, modCallback);
        }
    } finally {
        datasetLifecycleManager.close(resourceName);
    }
}

Example 4 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class MetadataNode method deleteTupleFromIndex.

private void deleteTupleFromIndex(JobId jobId, IMetadataIndex metadataIndex, ITupleReference tuple) throws ACIDException, HyracksDataException {
    long resourceID = metadataIndex.getResourceId();
    String resourceName = metadataIndex.getFile().getRelativePath();
    ILSMIndex lsmIndex = (ILSMIndex) datasetLifecycleManager.get(resourceName);
    try {
        datasetLifecycleManager.open(resourceName);
        // prepare a Callback for logging
        IModificationOperationCallback modCallback = createIndexModificationCallback(jobId, resourceID, metadataIndex, lsmIndex, Operation.DELETE);
        ILSMIndexAccessor indexAccessor = lsmIndex.createAccessor(modCallback, NoOpOperationCallback.INSTANCE);
        ITransactionContext txnCtx = transactionSubsystem.getTransactionManager().getTransactionContext(jobId, false);
        txnCtx.setWriteTxn(true);
        txnCtx.registerIndexAndCallback(resourceID, lsmIndex, (AbstractOperationCallback) modCallback, metadataIndex.isPrimaryIndex());
        LSMIndexUtil.checkAndSetFirstLSN((AbstractLSMIndex) lsmIndex, transactionSubsystem.getLogManager());
        indexAccessor.forceDelete(tuple);
        //by the job commit log event
        if (!((TransactionContext) txnCtx).getPrimaryIndexOpTracker().equals(lsmIndex.getOperationTracker())) {
            lsmIndex.getOperationTracker().completeOperation(lsmIndex, LSMOperationType.FORCE_MODIFICATION, null, modCallback);
        }
    } finally {
        datasetLifecycleManager.close(resourceName);
    }
}

Example 5 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMInvertedIndex method createMergeOperation.

@Override
protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException {
    ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(getLsmHarness(), opCtx);
    IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(opCtx);
    return new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
}