Examples with ILSMDiskComponentBulkLoader org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader used on opensource projects

Example 6 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMBTree method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeFlushOperation flushOp = (LSMBTreeFlushOperation) operation;
    LSMBTreeMemoryComponent flushingComponent = (LSMBTreeMemoryComponent) flushOp.getFlushingComponent();
    IIndexAccessor accessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        IIndexCursor countingCursor = ((BTreeAccessor) accessor).createCountingSearchCursor();
        accessor.search(countingCursor, nullPred);
        try {
            while (countingCursor.hasNext()) {
                countingCursor.next();
                ITupleReference countTuple = countingCursor.getTuple();
                numElements = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
            }
        } finally {
            countingCursor.close();
        }
    }
    LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
    IIndexCursor scanCursor = accessor.createSearchCursor(false);
    accessor.search(scanCursor, nullPred);
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            componentBulkLoader.add(scanCursor.getTuple());
        }
    } finally {
        scanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getBTree());
    }
    // Write metadata from memory component to disk
    // Q. what about the merge operation? how do we resolve conflicts
    // A. Through providing an appropriate ILSMIOOperationCallback
    // Must not reset the metadata before the flush is completed
    // Use the copy of the metadata in the opContext
    // TODO This code should be in the callback and not in the index
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}

Example 7 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMBTree method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rangePred);
    List<ILSMComponent> mergedComponents = mergeOp.getMergingComponents();
    long numElements = 0L;
    if (hasBloomFilter) {
        //count elements in btree for creating Bloomfilter
        for (int i = 0; i < mergedComponents.size(); ++i) {
            numElements += ((LSMBTreeDiskComponent) mergedComponents.get(i)).getBloomFilter().getNumElements();
        }
    }
    LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (mergedComponent.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
    }
    componentBulkLoader.end();
    return mergedComponent;
}

Example 8 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMRTreeWithAntiMatterTuples method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
    // Renaming order is critical because we use assume ordering when we
    // read the file names when we open the tree.
    // The RTree should be renamed before the BTree.
    LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent();
    ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
    SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
    memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
    LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), null, null, true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
    // Since the LSM-RTree is used as a secondary assumption, the
    // primary key will be the last comparator in the BTree comparators
    TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(), linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), flushingComponent.getRTree().getBufferCache(), comparatorFields);
    boolean isEmpty = true;
    try {
        while (rtreeScanCursor.hasNext()) {
            isEmpty = false;
            rtreeScanCursor.next();
            rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
        }
    } finally {
        rtreeScanCursor.close();
    }
    if (!isEmpty) {
        rTreeTupleSorter.sort();
    }
    // scan the memory BTree
    ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    BTreeRangeSearchCursor btreeScanCursor = (BTreeRangeSearchCursor) memBTreeAccessor.createSearchCursor(false);
    RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
    memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
    TreeTupleSorter bTreeTupleSorter = new TreeTupleSorter(flushingComponent.getBTree().getFileId(), linearizerArray, btreeLeafFrameFactory.createFrame(), btreeLeafFrameFactory.createFrame(), flushingComponent.getBTree().getBufferCache(), comparatorFields);
    isEmpty = true;
    try {
        while (btreeScanCursor.hasNext()) {
            isEmpty = false;
            btreeScanCursor.next();
            bTreeTupleSorter.insertTupleEntry(btreeScanCursor.getPageId(), btreeScanCursor.getTupleOffset());
        }
    } finally {
        btreeScanCursor.close();
    }
    if (!isEmpty) {
        bTreeTupleSorter.sort();
    }
    LSMRTreeWithAntiMatterTuplesFlushCursor cursor = new LSMRTreeWithAntiMatterTuplesFlushCursor(rTreeTupleSorter, bTreeTupleSorter, comparatorFields, linearizerArray);
    cursor.open(null, null);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
    }
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}