Examples with ILSMIndexAccessor org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor used on opensource projects

Example 26 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class AbstractLSMIndex method flushMemoryComponents.

// What if more than one memory component needs flushing??
protected void flushMemoryComponents() throws HyracksDataException {
    BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
    ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    accessor.scheduleFlush(cb);
    try {
        cb.waitForIO();
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw HyracksDataException.create(e);
    }
}

Example 27 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMIndexInsertUpdateDeleteOperatorNodePushable method nextFrame.

@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
    accessor.reset(buffer);
    ILSMIndexAccessor lsmAccessor = (ILSMIndexAccessor) indexAccessor;
    int nextFlushTupleIndex = 0;
    int tupleCount = accessor.getTupleCount();
    for (int i = 0; i < tupleCount; i++) {
        try {
            if (tupleFilter != null) {
                frameTuple.reset(accessor, i);
                if (!tupleFilter.accept(frameTuple)) {
                    continue;
                }
            }
            tuple.reset(accessor, i);
            switch(op) {
                case INSERT:
                    {
                        if (!lsmAccessor.tryInsert(tuple)) {
                            flushPartialFrame(nextFlushTupleIndex, i);
                            nextFlushTupleIndex = i;
                            lsmAccessor.insert(tuple);
                        }
                        break;
                    }
                case DELETE:
                    {
                        if (!lsmAccessor.tryDelete(tuple)) {
                            flushPartialFrame(nextFlushTupleIndex, i);
                            nextFlushTupleIndex = i;
                            lsmAccessor.delete(tuple);
                        }
                        break;
                    }
                case UPSERT:
                    {
                        if (!lsmAccessor.tryUpsert(tuple)) {
                            flushPartialFrame(nextFlushTupleIndex, i);
                            nextFlushTupleIndex = i;
                            lsmAccessor.upsert(tuple);
                        }
                        break;
                    }
                case UPDATE:
                    {
                        if (!lsmAccessor.tryUpdate(tuple)) {
                            flushPartialFrame(nextFlushTupleIndex, i);
                            nextFlushTupleIndex = i;
                            lsmAccessor.update(tuple);
                        }
                        break;
                    }
                default:
                    {
                        throw new HyracksDataException("Unsupported operation " + op + " in tree index InsertUpdateDelete operator");
                    }
            }
        } catch (HyracksDataException e) {
            throw e;
        } catch (Exception e) {
            throw new HyracksDataException(e);
        }
    }
    if (nextFlushTupleIndex == 0) {
        // No partial flushing was necessary. Forward entire frame.
        writeBuffer.ensureFrameSize(buffer.capacity());
        FrameUtils.copyAndFlip(buffer, writeBuffer.getBuffer());
        FrameUtils.flushFrame(writeBuffer.getBuffer(), writer);
    } else {
        // Flush remaining partial frame.
        flushPartialFrame(nextFlushTupleIndex, tupleCount);
    }
}

Example 28 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMRTreeMergeTestDriver method runTest.

@Override
protected void runTest(ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, ITupleReference key, RTreePolicyType rtreePolicyType) throws Exception {
    AbstractRTreeTestContext ctx = createTestContext(fieldSerdes, valueProviderFactories, numKeys, rtreePolicyType);
    ctx.getIndex().create();
    ctx.getIndex().activate();
    // to determine which field types to generate.
    if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
        rTreeTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
    } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
        rTreeTestUtils.bulkLoadDoubleTuples(ctx, numTuplesToInsert, getRandom());
    }
    int maxTreesToMerge = AccessMethodTestsConfig.LSM_RTREE_BULKLOAD_ROUNDS;
    for (int i = 0; i < maxTreesToMerge; i++) {
        for (int j = 0; j < i; j++) {
            if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
                rTreeTestUtils.insertIntTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            } else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
                rTreeTestUtils.insertDoubleTuples(ctx, numTuplesToInsert, getRandom());
                // Deactivate and the re-activate the index to force it flush its in memory component
                ctx.getIndex().deactivate();
                ctx.getIndex().activate();
            }
        }
        ILSMIndexAccessor accessor = (ILSMIndexAccessor) ctx.getIndexAccessor();
        accessor.scheduleMerge(NoOpIOOperationCallbackFactory.INSTANCE.createIoOpCallback(), ((AbstractLSMRTree) ctx.getIndex()).getImmutableComponents());
        rTreeTestUtils.checkScan(ctx);
        rTreeTestUtils.checkDiskOrderScan(ctx);
        rTreeTestUtils.checkRangeSearch(ctx, key);
    }
    ctx.getIndex().deactivate();
    ctx.getIndex().destroy();
}

Example 29 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class CorrelatedPrefixMergePolicyTest method mockIndex.

private IndexInfo mockIndex(boolean isPrimary, List<ILSMDiskComponentId> componentIDs, List<ILSMDiskComponentId> resultComponentIDs, int partition) throws HyracksDataException {
    List<ILSMDiskComponent> components = new ArrayList<>();
    for (ILSMDiskComponentId id : componentIDs) {
        ILSMDiskComponent component = Mockito.mock(ILSMDiskComponent.class);
        Mockito.when(component.getComponentId()).thenReturn(id);
        Mockito.when(component.getComponentSize()).thenReturn(DEFAULT_COMPONENT_SIZE);
        Mockito.when(component.getState()).thenReturn(ComponentState.READABLE_UNWRITABLE);
        components.add(component);
    }
    ILSMIndex index = Mockito.mock(ILSMIndex.class);
    Mockito.when(index.getImmutableComponents()).thenReturn(components);
    ILSMIndexAccessor accessor = Mockito.mock(ILSMIndexAccessor.class);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            List<ILSMDiskComponent> mergedComponents = invocation.getArgumentAt(1, List.class);
            mergedComponents.forEach(component -> {
                try {
                    resultComponentIDs.add(component.getComponentId());
                } catch (HyracksDataException e) {
                    e.printStackTrace();
                }
            });
            return null;
        }
    }).when(accessor).scheduleMerge(Mockito.any(ILSMIOOperationCallback.class), Mockito.anyListOf(ILSMDiskComponent.class));
    Mockito.when(index.createAccessor(Mockito.any(IModificationOperationCallback.class), Mockito.any(ISearchOperationCallback.class))).thenReturn(accessor);
    Mockito.when(index.isPrimaryIndex()).thenReturn(isPrimary);
    return new IndexInfo(index, DATASET_ID, 0, partition);
}

Example 30 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMRTree method createMergeOperation.

@Override
protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx, List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs, ILSMIOOperationCallback callback) throws HyracksDataException {
    ITreeIndexCursor cursor = new LSMRTreeSortedCursor(opCtx, linearizer, buddyBTreeFields);
    ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields);
    return new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
}