Search in sources :

Example 1 with LSMBTreeCopyTupleWriterFactory

use of org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory in project asterixdb by apache.

the class LSMBTreeUtil method createExternalBTree.

public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
    LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, true);
    LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length);
    ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
    ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
    // This is the tuple writer that can do both inserts and deletes
    LSMBTreeTupleWriterFactory transactionTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    // This is the leaf frame factory for transaction components since it
    // can be used for both inserts and deletes
    ITreeIndexFrameFactory transactionLeafFrameFactory = new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
    TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    // This is the component factory for transactions
    TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length);
    //TODO remove BloomFilter from external dataset's secondary LSMBTree index
    ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true);
    // the disk only index uses an empty ArrayList for virtual buffer caches
    ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, durable);
    return lsmTree;
}
Also used : BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMBTreeCopyTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) LSMBTreeFileManager(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager) LSMBTreeTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory)

Example 2 with LSMBTreeCopyTupleWriterFactory

use of org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory in project asterixdb by apache.

the class LSMBTreeUtil method createExternalBTreeWithBuddy.

public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
    ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
    IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length];
    for (int i = 0; i < buddyBtreeTypeTraits.length; i++) {
        buddyBtreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
        buddyBtreeCmpFactories[i] = cmpFactories[buddyBTreeFields[i]];
    }
    TypeAwareTupleWriterFactory buddyBtreeTupleWriterFactory = new TypeAwareTupleWriterFactory(buddyBtreeTypeTraits);
    ITreeIndexFrameFactory buddyBtreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(buddyBtreeTupleWriterFactory);
    ITreeIndexFrameFactory buddyBtreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(buddyBtreeTupleWriterFactory);
    LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length);
    ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
    TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
    int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length];
    for (int i = 0; i < buddyBtreeCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    // buddy b-tree factory
    TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory, buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
    ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, diskBuddyBTreeFactory);
    // the disk only index uses an empty ArrayList for virtual buffer caches
    ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory, insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, buddyBtreeCmpFactories, buddyBTreeFields, durable);
    return lsmTree;
}
Also used : LSMBTreeCopyTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) LSMBTreeTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) LSMBTreeWithBuddyFileManager(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) ExternalBTreeWithBuddy(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory)

Example 3 with LSMBTreeCopyTupleWriterFactory

use of org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory in project asterixdb by apache.

the class LSMBTreeUtil method createLSMTree.

public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
    LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, true);
    LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length);
    LSMBTreeTupleWriterFactory bulkLoadTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
    ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
    ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
    TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
    BloomFilterFactory bloomFilterFactory = needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields) : null;
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    //Primary LSMBTree index has a BloomFilter.
    ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, needKeyDupCheck);
    return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields, filterFields, durable);
}
Also used : LSMBTreeCopyTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) LSMComponentFilterFrameFactory(org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory) LSMBTreeFileManager(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager) LSMBTreeTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory) LSMComponentFilterManager(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) ComponentFilterHelper(org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper)

Aggregations

BloomFilterFactory (org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory)3 BTreeNSMInteriorFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory)3 BTreeNSMLeafFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory)3 BTree (org.apache.hyracks.storage.am.btree.impls.BTree)3 ITreeIndexFrameFactory (org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory)3 ExternalBTree (org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree)3 LSMBTree (org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree)3 LSMBTreeCopyTupleWriterFactory (org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory)3 LSMBTreeTupleWriterFactory (org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory)3 ILSMIndexFileManager (org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager)3 BTreeFactory (org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory)3 TypeAwareTupleWriterFactory (org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory)2 LSMBTreeFileManager (org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager)2 IBinaryComparatorFactory (org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory)1 ITypeTraits (org.apache.hyracks.api.dataflow.value.ITypeTraits)1 ExternalBTreeWithBuddy (org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy)1 LSMBTreeWithBuddyFileManager (org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager)1 LSMComponentFilterFrameFactory (org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory)1 ComponentFilterHelper (org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper)1 LSMComponentFilterManager (org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager)1