Examples with ILSMIndexAccessor org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor used on opensource projects

Example 21 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class ExternalBTreeWithBuddy method scheduleMerge.

@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
    ILSMIndexOperationContext bctx = createOpContext(NoOpOperationCallback.INSTANCE, 0);
    bctx.setOperation(IndexOperation.MERGE);
    List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
    ITreeIndexCursor cursor = new LSMBTreeWithBuddySortedCursor(bctx, buddyBTreeFields);
    LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
    ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), bctx, opCtx -> new LSMBTreeWithBuddySearchCursor(opCtx, buddyBTreeFields));
    // Since we have two lists of components, to tell whether we need to
    // keep deleted tuples, we need to know
    // which list to check against and we need to synchronize for this
    boolean keepDeleteTuples = false;
    if (version == 0) {
        keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1);
    } else {
        keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != secondDiskComponents.get(secondDiskComponents.size() - 1);
    }
    ioScheduler.scheduleOperation(new LSMBTreeWithBuddyMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir(), keepDeleteTuples));
}

Example 22 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMBTree method createMergeOperation.

@Override
protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) {
    boolean returnDeletedTuples = false;
    ILSMIndexAccessor accessor = createAccessor(opCtx);
    if (mergingComponents.get(mergingComponents.size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
        returnDeletedTuples = true;
    }
    ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples);
    return new LSMBTreeMergeOperation(accessor, mergingComponents, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
}

Example 23 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class RecoveryManager method redo.

private static void redo(ILogRecord logRecord, IDatasetLifecycleManager datasetLifecycleManager) {
    try {
        int datasetId = logRecord.getDatasetId();
        long resourceId = logRecord.getResourceId();
        ILSMIndex index = (ILSMIndex) datasetLifecycleManager.getIndex(datasetId, resourceId);
        ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.INSERT_BYTE) {
            indexAccessor.forceInsert(logRecord.getNewValue());
        } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.DELETE_BYTE) {
            indexAccessor.forceDelete(logRecord.getNewValue());
        } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.UPSERT_BYTE) {
            // redo, upsert the new value
            indexAccessor.forceUpsert(logRecord.getNewValue());
        } else {
            throw new IllegalStateException("Unsupported OperationType: " + logRecord.getNewOp());
        }
    } catch (Exception e) {
        throw new IllegalStateException("Failed to redo", e);
    }
}

Example 24 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMBTreeMergeTestDriver method runTest.

@Override
protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
    OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys, leafType, false);
    ctx.getIndex().create();
    ctx.getIndex().activate();
    // to determine which field types to generate.
    if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
    } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
        orderedIndexTestUtils.bulkLoadStringTuples(ctx, numTuplesToInsert, getRandom());
    }
    int maxTreesToMerge = AccessMethodTestsConfig.LSM_BTREE_MAX_TREES_TO_MERGE;
    for (int i = 0; i < maxTreesToMerge; i++) {
        for (int j = 0; j < i; j++) {
            if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
                orderedIndexTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
                orderedIndexTestUtils.insertStringTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            }
        }
        ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
        accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), ((LSMBTree) ctx.getIndex()).getImmutableComponents());
        orderedIndexTestUtils.checkPointSearches(ctx);
        orderedIndexTestUtils.checkScan(ctx);
        orderedIndexTestUtils.checkDiskOrderScan(ctx);
        orderedIndexTestUtils.checkRangeSearch(ctx, lowKey, highKey, true, true);
        if (prefixLowKey != null && prefixHighKey != null) {
            orderedIndexTestUtils.checkRangeSearch(ctx, prefixLowKey, prefixHighKey, true, true);
        }
    }
    ctx.getIndex().deactivate();
    ctx.getIndex().destroy();
}

Example 25 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMBTreeModificationOperationCallbackTest method modificationCallbackTest.

@Override
@Test
public void modificationCallbackTest() throws Exception {
    ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(cb, NoOpOperationCallback.INSTANCE);
    for (int j = 0; j < 2; j++) {
        isFoundNull = true;
        for (int i = 0; i < NUM_TUPLES; i++) {
            TupleUtils.createIntegerTuple(builder, tuple, i);
            accessor.insert(tuple);
        }
        if (j == 1) {
            accessor.scheduleFlush(ioOpCallback);
            ioOpCallback.waitForIO();
            isFoundNull = true;
        } else {
            isFoundNull = false;
        }
        for (int i = 0; i < NUM_TUPLES; i++) {
            TupleUtils.createIntegerTuple(builder, tuple, i);
            accessor.upsert(tuple);
        }
        if (j == 1) {
            accessor.scheduleFlush(ioOpCallback);
            ioOpCallback.waitForIO();
            isFoundNull = true;
        } else {
            isFoundNull = false;
        }
        for (int i = 0; i < NUM_TUPLES; i++) {
            TupleUtils.createIntegerTuple(builder, tuple, i);
            accessor.delete(tuple);
        }
        accessor.scheduleFlush(ioOpCallback);
        ioOpCallback.waitForIO();
    }
}