Search in sources :

Example 1 with LSMInvertedIndex

use of org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex in project asterixdb by apache.

the class InvertedIndexUtils method createLSMInvertedIndex.

public static LSMInvertedIndex createLSMInvertedIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, IFileMapProvider diskFileMapProvider, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IBinaryTokenizerFactory tokenizerFactory, IBufferCache diskBufferCache, String absoluteOnDiskDir, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable, IMetadataPageManagerFactory pageManagerFactory) throws HyracksDataException {
    BTreeFactory deletedKeysBTreeFactory = createDeletedKeysBTreeFactory(ioManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, diskBufferCache, pageManagerFactory);
    int[] bloomFilterKeyFields = new int[invListCmpFactories.length];
    for (int i = 0; i < invListCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    FileReference onDiskDirFileRef = ioManager.resolveAbsolutePath(absoluteOnDiskDir);
    LSMInvertedIndexFileManager fileManager = new LSMInvertedIndexFileManager(ioManager, diskFileMapProvider, onDiskDirFileRef, deletedKeysBTreeFactory);
    IInvertedListBuilderFactory invListBuilderFactory = new FixedSizeElementInvertedListBuilderFactory(invListTypeTraits);
    OnDiskInvertedIndexFactory invIndexFactory = new OnDiskInvertedIndexFactory(ioManager, diskBufferCache, diskFileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, fileManager, pageManagerFactory);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    return new LSMInvertedIndex(ioManager, virtualBufferCaches, invIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable);
}
Also used : TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) PartitionedLSMInvertedIndex(org.apache.hyracks.storage.am.lsm.invertedindex.impls.PartitionedLSMInvertedIndex) LSMInvertedIndex(org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex) LSMComponentFilterFrameFactory(org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory) LSMComponentFilterManager(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager) LSMInvertedIndexFileManager(org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) FixedSizeElementInvertedListBuilderFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeElementInvertedListBuilderFactory) PartitionedOnDiskInvertedIndexFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory) OnDiskInvertedIndexFactory(org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) FileReference(org.apache.hyracks.api.io.FileReference) ComponentFilterHelper(org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper) IInvertedListBuilderFactory(org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory)

Aggregations

FileReference (org.apache.hyracks.api.io.FileReference)1 BloomFilterFactory (org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory)1 TypeAwareTupleWriterFactory (org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory)1 LSMComponentFilterFrameFactory (org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory)1 BTreeFactory (org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory)1 ComponentFilterHelper (org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper)1 LSMComponentFilterManager (org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager)1 IInvertedListBuilderFactory (org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilderFactory)1 LSMInvertedIndex (org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndex)1 LSMInvertedIndexFileManager (org.apache.hyracks.storage.am.lsm.invertedindex.impls.LSMInvertedIndexFileManager)1 PartitionedLSMInvertedIndex (org.apache.hyracks.storage.am.lsm.invertedindex.impls.PartitionedLSMInvertedIndex)1 FixedSizeElementInvertedListBuilderFactory (org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.FixedSizeElementInvertedListBuilderFactory)1 OnDiskInvertedIndexFactory (org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndexFactory)1 PartitionedOnDiskInvertedIndexFactory (org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.PartitionedOnDiskInvertedIndexFactory)1