Examples with ILSMDiskComponentBulkLoader org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader used on opensource projects

Example 1 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMRTreeWithAntiMatterTuples method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    MergeOperation mergeOp = (MergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rtreeSearchPred);
    // Bulk load the tuples from all on-disk RTrees into the new RTree.
    LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
    }
    componentBulkLoader.end();
    return component;
}

Example 2 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMRTree method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
    ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    search(opCtx, cursor, rtreeSearchPred);
    LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader;
    // lsmHarness.endSearch() is called once when the r-trees have been merged.
    if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
        // Keep the deleted tuples since the oldest disk component is not included in the merge operation
        long numElements = 0L;
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
        }
        componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
        LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx);
        search(opCtx, btreeCursor, rtreeSearchPred);
        try {
            while (btreeCursor.hasNext()) {
                btreeCursor.next();
                ITupleReference tuple = btreeCursor.getTuple();
                componentBulkLoader.delete(tuple);
            }
        } finally {
            btreeCursor.close();
        }
    } else {
        //no buddy-btree needed
        componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false);
    }
    //search old rtree components
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference frameTuple = cursor.getTuple();
            componentBulkLoader.add(frameTuple);
        }
    } finally {
        cursor.close();
    }
    if (mergedComponent.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
            filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
        }
        getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
    }
    componentBulkLoader.end();
    return mergedComponent;
}

Example 3 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMRTree method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMRTreeFlushOperation flushOp = (LSMRTreeFlushOperation) operation;
    LSMRTreeMemoryComponent flushingComponent = (LSMRTreeMemoryComponent) flushOp.getFlushingComponent();
    // Renaming order is critical because we use assume ordering when we
    // read the file names when we open the tree.
    // The RTree should be renamed before the BTree.
    // scan the memory RTree
    ITreeIndexAccessor memRTreeAccessor = flushingComponent.getRTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RTreeSearchCursor rtreeScanCursor = (RTreeSearchCursor) memRTreeAccessor.createSearchCursor(false);
    SearchPredicate rtreeNullPredicate = new SearchPredicate(null, null);
    memRTreeAccessor.search(rtreeScanCursor, rtreeNullPredicate);
    LSMRTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true);
    //count the number of tuples in the buddy btree
    ITreeIndexAccessor memBTreeAccessor = flushingComponent.getBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    RangePredicate btreeNullPredicate = new RangePredicate(null, null, true, true, null, null);
    IIndexCursor btreeCountingCursor = ((BTreeAccessor) memBTreeAccessor).createCountingSearchCursor();
    memBTreeAccessor.search(btreeCountingCursor, btreeNullPredicate);
    long numBTreeTuples = 0L;
    try {
        while (btreeCountingCursor.hasNext()) {
            btreeCountingCursor.next();
            ITupleReference countTuple = btreeCountingCursor.getTuple();
            numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
        }
    } finally {
        btreeCountingCursor.close();
    }
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
    ITreeIndexCursor cursor;
    IBinaryComparatorFactory[] linearizerArray = { linearizer };
    TreeTupleSorter rTreeTupleSorter = new TreeTupleSorter(flushingComponent.getRTree().getFileId(), linearizerArray, rtreeLeafFrameFactory.createFrame(), rtreeLeafFrameFactory.createFrame(), flushingComponent.getRTree().getBufferCache(), comparatorFields);
    // BulkLoad the tuples from the in-memory tree into the new disk
    // RTree.
    boolean isEmpty = true;
    try {
        while (rtreeScanCursor.hasNext()) {
            isEmpty = false;
            rtreeScanCursor.next();
            rTreeTupleSorter.insertTupleEntry(rtreeScanCursor.getPageId(), rtreeScanCursor.getTupleOffset());
        }
    } finally {
        rtreeScanCursor.close();
    }
    rTreeTupleSorter.sort();
    cursor = rTreeTupleSorter;
    if (!isEmpty) {
        try {
            while (cursor.hasNext()) {
                cursor.next();
                ITupleReference frameTuple = cursor.getTuple();
                componentBulkLoader.add(frameTuple);
            }
        } finally {
            cursor.close();
        }
    }
    // scan the memory BTree
    IIndexCursor btreeScanCursor = memBTreeAccessor.createSearchCursor(false);
    memBTreeAccessor.search(btreeScanCursor, btreeNullPredicate);
    try {
        while (btreeScanCursor.hasNext()) {
            btreeScanCursor.next();
            ITupleReference frameTuple = btreeScanCursor.getTuple();
            componentBulkLoader.delete(frameTuple);
        }
    } finally {
        btreeScanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
    }
    // Note. If we change the filter to write to metadata object, we don't need the if block above
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}

Example 4 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMInvertedIndex method merge.

@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
    LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
    IIndexCursor cursor = mergeOp.getCursor();
    RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
    ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
    opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
    // Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
    search(opCtx, cursor, mergePred);
    // Create an inverted index instance.
    LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(), mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
    ILSMDiskComponentBulkLoader componentBulkLoader;
    // lsmHarness.endSearch() is called once when the inverted indexes have been merged.
    if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
        // Keep the deleted tuples since the oldest disk component is not included in the merge operation
        LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
        search(opCtx, btreeCursor, mergePred);
        long numElements = 0L;
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
        }
        componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
        try {
            while (btreeCursor.hasNext()) {
                btreeCursor.next();
                ITupleReference tuple = btreeCursor.getTuple();
                componentBulkLoader.delete(tuple);
            }
        } finally {
            btreeCursor.close();
        }
    } else {
        componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
    }
    try {
        while (cursor.hasNext()) {
            cursor.next();
            ITupleReference tuple = cursor.getTuple();
            componentBulkLoader.add(tuple);
        }
    } finally {
        cursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
            ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
            ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
            if (min != null) {
                filterTuples.add(min);
            }
            if (max != null) {
                filterTuples.add(max);
            }
        }
        getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
        getFilterManager().writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
    }
    componentBulkLoader.end();
    return component;
}

Example 5 with ILSMDiskComponentBulkLoader

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentBulkLoader in project asterixdb by apache.

the class LSMInvertedIndex method flush.

@Override
public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
    LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
    // Create an inverted index instance to be bulk loaded.
    LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, flushOp.getTarget(), flushOp.getDeletedKeysBTreeTarget(), flushOp.getBloomFilterTarget(), true);
    // Create a scan cursor on the BTree underlying the in-memory inverted index.
    LSMInvertedIndexMemoryComponent flushingComponent = (LSMInvertedIndexMemoryComponent) flushOp.getFlushingComponent();
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    // Search the deleted keys BTree to calculate the number of elements for BloomFilter
    IIndexAccessor deletedKeysBTreeAccessor = flushingComponent.getDeletedKeysBTree().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    IIndexCursor btreeCountingCursor = ((BTreeAccessor) deletedKeysBTreeAccessor).createCountingSearchCursor();
    deletedKeysBTreeAccessor.search(btreeCountingCursor, nullPred);
    long numBTreeTuples = 0L;
    try {
        while (btreeCountingCursor.hasNext()) {
            btreeCountingCursor.next();
            ITupleReference countTuple = btreeCountingCursor.getTuple();
            numBTreeTuples = IntegerPointable.getInteger(countTuple.getFieldData(0), countTuple.getFieldStart(0));
        }
    } finally {
        btreeCountingCursor.close();
    }
    ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numBTreeTuples, false, false);
    // Create a scan cursor on the deleted keys BTree underlying the in-memory inverted index.
    IIndexCursor deletedKeysScanCursor = deletedKeysBTreeAccessor.createSearchCursor(false);
    deletedKeysBTreeAccessor.search(deletedKeysScanCursor, nullPred);
    try {
        while (deletedKeysScanCursor.hasNext()) {
            deletedKeysScanCursor.next();
            ((LSMInvertedIndexDiskComponentBulkLoader) componentBulkLoader).delete(deletedKeysScanCursor.getTuple());
        }
    } finally {
        deletedKeysScanCursor.close();
    }
    // Scan the in-memory inverted index
    InMemoryInvertedIndexAccessor memInvIndexAccessor = (InMemoryInvertedIndexAccessor) flushingComponent.getInvIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    BTreeAccessor memBTreeAccessor = memInvIndexAccessor.getBTreeAccessor();
    IIndexCursor scanCursor = memBTreeAccessor.createSearchCursor(false);
    memBTreeAccessor.search(scanCursor, nullPred);
    // Bulk load the disk inverted index from the in-memory inverted index.
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            componentBulkLoader.add(scanCursor.getTuple());
        }
    } finally {
        scanCursor.close();
    }
    if (component.getLSMComponentFilter() != null) {
        List<ITupleReference> filterTuples = new ArrayList<>();
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
        filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
        filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
        filterManager.writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
    }
    flushingComponent.getMetadata().copy(component.getMetadata());
    componentBulkLoader.end();
    return component;
}