Search in sources :

Example 6 with BloomFilterFactory

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory in project asterixdb by apache.

the class InvertedIndexUtils method createLSMInvertedIndex.

public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException {
    BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
    int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
    for (int i = 0; i < invListCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
    LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory);
    IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
    OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}
Also used : TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) PartitionedLSMInvertedIndex(org.apache.hyracks.storage.am.lsm.invertedindex.impls.PartitionedLSMInvertedIndex) LSMInvertedIndex(org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex) LSMComponentFilterFrameFactory(org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory) LSMComponentFilterManager(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager) LSMInvertedIndexFileManager(org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) FixedSizeElementInvertedListBuilderFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeElementInvertedListBuilderFactory) PartitionedOnDiskInvertedIndexFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory) OnDiskInvertedIndexFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) FileReference(org.apache.hyracks.api.io.FileReference) ComponentFilterHelper(org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper) IInvertedListBuilderFactory(org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory)

Example 7 with BloomFilterFactory

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory in project asterixdb by apache.

the class InvertedIndexUtils method createPartitionedLSMInvertedIndex.

public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IPageManagerFactory pageManagerFactory) throws HyracksDataException {
    BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
    int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
    for (int i = 0; i < invListCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
    LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory);
    IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
    PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}
Also used : TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) LSMComponentFilterFrameFactory(org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory) LSMComponentFilterManager(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager) PartitionedLSMInvertedIndex(org.apache.hyracks.storage.am.lsm.invertedindex.impls.PartitionedLSMInvertedIndex) LSMInvertedIndexFileManager(org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) FixedSizeElementInvertedListBuilderFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeElementInvertedListBuilderFactory) PartitionedOnDiskInvertedIndexFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) FileReference(org.apache.hyracks.api.io.FileReference) ComponentFilterHelper(org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper) IInvertedListBuilderFactory(org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory)

Aggregations

BloomFilterFactory (org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory)7 BTreeFactory (org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory)7 BTreeNSMInteriorFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory)5 BTreeNSMLeafFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory)5 BTree (org.apache.hyracks.storage.am.btree.impls.BTree)5 ITreeIndexFrameFactory (org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory)5 TypeAwareTupleWriterFactory (org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory)5 ILSMIndexFileManager (org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager)5 LSMComponentFilterFrameFactory (org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory)4 ComponentFilterHelper (org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper)4 LSMComponentFilterManager (org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager)4 IBinaryComparatorFactory (org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory)3 ITypeTraits (org.apache.hyracks.api.dataflow.value.ITypeTraits)3 ExternalBTree (org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree)3 LSMBTree (org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree)3 LSMBTreeCopyTupleWriterFactory (org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory)3 LSMBTreeTupleWriterFactory (org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory)3 FileReference (org.apache.hyracks.api.io.FileReference)2 ITreeIndexTupleWriterFactory (org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory)2 LSMBTreeFileManager (org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager)2