Examples with BloomFilterFactory org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory used on opensource projects

Example 6 with BloomFilterFactory

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory in project asterixdb by apache.

the class InvertedIndexUtils method createLSMInvertedIndex.

public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException {
    BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
    int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
    for (int i = 0; i < invListCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
    LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory);
    IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
    OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}

Example 7 with BloomFilterFactory

use of org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory in project asterixdb by apache.

the class InvertedIndexUtils method createPartitionedLSMInvertedIndex.

public static PartitionedLSMInvertedIndex createPartitionedLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IPageManagerFactory pageManagerFactory) throws HyracksDataException {
    BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
    int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
    for (int i = 0; i < invListCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
    LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory);
    IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
    PartitionedOnDiskInvertedIndexFactory invIndexFactory = new PartitionedOnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    return new PartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}