Search in sources :

Example 1 with ILSMIndexFileManager

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager in project asterixdb by apache.

the class LSMBTreeUtil method createExternalBTree.

public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
    LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, true);
    LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length);
    ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
    ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
    // This is the tuple writer that can do both inserts and deletes
    LSMBTreeTupleWriterFactory transactionTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    // This is the leaf frame factory for transaction components since it
    // can be used for both inserts and deletes
    ITreeIndexFrameFactory transactionLeafFrameFactory = new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
    TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    // This is the component factory for transactions
    TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length);
    //TODO remove BloomFilter from external dataset's secondary LSMBTree index
    ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true);
    // the disk only index uses an empty ArrayList for virtual buffer caches
    ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, durable);
    return lsmTree;
}
Also used : BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMBTreeCopyTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) LSMBTreeFileManager(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeFileManager) LSMBTreeTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory)

Example 2 with ILSMIndexFileManager

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager in project asterixdb by apache.

the class LSMBTreeUtil method createExternalBTreeWithBuddy.

public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
    ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
    IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length];
    for (int i = 0; i < buddyBtreeTypeTraits.length; i++) {
        buddyBtreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
        buddyBtreeCmpFactories[i] = cmpFactories[buddyBTreeFields[i]];
    }
    TypeAwareTupleWriterFactory buddyBtreeTupleWriterFactory = new TypeAwareTupleWriterFactory(buddyBtreeTypeTraits);
    ITreeIndexFrameFactory buddyBtreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(buddyBtreeTupleWriterFactory);
    ITreeIndexFrameFactory buddyBtreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(buddyBtreeTupleWriterFactory);
    LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length);
    ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
    TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
    int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length];
    for (int i = 0; i < buddyBtreeCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    // buddy b-tree factory
    TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory, buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
    ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, diskBuddyBTreeFactory);
    // the disk only index uses an empty ArrayList for virtual buffer caches
    ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory, insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, buddyBtreeCmpFactories, buddyBTreeFields, durable);
    return lsmTree;
}
Also used : LSMBTreeCopyTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) LSMBTree(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree) ExternalBTree(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree) LSMBTreeTupleWriterFactory(org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) LSMBTreeWithBuddyFileManager(org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTreeWithBuddyFileManager) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) ExternalBTreeWithBuddy(org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory)

Example 3 with ILSMIndexFileManager

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager in project asterixdb by apache.

the class LSMIndexFileManagerTest method cleanInvalidFilesTest.

public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
    String dirPath = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree" + simpleDateFormat.format(new Date()) + sep;
    File f = new File(dirPath);
    f.mkdirs();
    FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
    ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
    fileManager.createDirs();
    List<FileReference> flushFiles = new ArrayList<>();
    List<FileReference> allFiles = new ArrayList<>();
    int numFileNames = 100;
    long sleepTime = 5;
    // Generate a bunch of flush files.
    for (int i = 0; i < numFileNames; i++) {
        LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
        flushFiles.add(relFlushFileRefs.getInsertIndexFileReference());
        Thread.sleep(sleepTime);
    }
    allFiles.addAll(flushFiles);
    // Simulate merging some of the flush files.
    // Merge range 0 to 4.
    FileReference mergeFile1 = simulateMerge(fileManager, flushFiles.get(0), flushFiles.get(4));
    allFiles.add(mergeFile1);
    // Merge range 5 to 9.
    FileReference mergeFile2 = simulateMerge(fileManager, flushFiles.get(5), flushFiles.get(9));
    allFiles.add(mergeFile2);
    // Merge range 10 to 19.
    FileReference mergeFile3 = simulateMerge(fileManager, flushFiles.get(10), flushFiles.get(19));
    allFiles.add(mergeFile3);
    // Merge range 20 to 29.
    FileReference mergeFile4 = simulateMerge(fileManager, flushFiles.get(20), flushFiles.get(29));
    allFiles.add(mergeFile4);
    // Merge range 50 to 79.
    FileReference mergeFile5 = simulateMerge(fileManager, flushFiles.get(50), flushFiles.get(79));
    allFiles.add(mergeFile5);
    // Simulate merging of merge files.
    FileReference mergeFile6 = simulateMerge(fileManager, mergeFile1, mergeFile2);
    allFiles.add(mergeFile6);
    FileReference mergeFile7 = simulateMerge(fileManager, mergeFile3, mergeFile4);
    allFiles.add(mergeFile7);
    // Create all files and set delete on exit for all files.
    for (FileReference fileRef : allFiles) {
        fileRef.getFile().createNewFile();
        fileRef.getFile().deleteOnExit();
    }
    // Populate expected valid flush files.
    List<String> expectedValidFiles = new ArrayList<>();
    for (int i = 30; i < 50; i++) {
        expectedValidFiles.add(flushFiles.get(i).getFile().getName());
    }
    for (int i = 80; i < 100; i++) {
        expectedValidFiles.add(flushFiles.get(i).getFile().getName());
    }
    // Populate expected valid merge files.
    expectedValidFiles.add(mergeFile5.getFile().getName());
    expectedValidFiles.add(mergeFile6.getFile().getName());
    expectedValidFiles.add(mergeFile7.getFile().getName());
    // Sort expected files.
    Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
    // Pass null and a dummy component finalizer. We don't test for physical consistency in this test.
    List<LSMComponentFileReferences> lsmComonentFileReference = fileManager.cleanupAndGetValidFiles();
    // Check actual files against expected files.
    assertEquals(expectedValidFiles.size(), lsmComonentFileReference.size());
    for (int i = 0; i < expectedValidFiles.size(); i++) {
        assertEquals(expectedValidFiles.get(i), lsmComonentFileReference.get(i).getInsertIndexFileReference().getFile().getName());
    }
    // Make sure invalid files were removed from the IODevices.
    ArrayList<String> remainingFiles = new ArrayList<>();
    File dir = new File(dirPath);
    FilenameFilter filter = new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return !name.startsWith(".");
        }
    };
    String[] files = dir.list(filter);
    for (String aFilePath : files) {
        File aFile = new File(aFilePath);
        remainingFiles.add(aFile.getName());
    }
    Collections.sort(remainingFiles, fileManager.getFileNameComparator());
    // Check actual files in directory against expected files.
    assertEquals(expectedValidFiles.size(), remainingFiles.size());
    for (int i = 0; i < expectedValidFiles.size(); i++) {
        assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
    }
}
Also used : ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) ArrayList(java.util.ArrayList) TestLsmIndexFileManager(org.apache.hyracks.storage.am.lsm.common.component.TestLsmIndexFileManager) Date(java.util.Date) FilenameFilter(java.io.FilenameFilter) FileReference(org.apache.hyracks.api.io.FileReference) File(java.io.File) LSMComponentFileReferences(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences)

Example 4 with ILSMIndexFileManager

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager in project asterixdb by apache.

the class LSMRTreeUtils method createLSMTreeWithAntiMatterTuples.

public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeComparatorFactories, IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizerCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
    ITreeIndexTupleWriterFactory rtreeInteriorFrameTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
    ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory;
    ITreeIndexTupleWriterFactory rtreeLeafFrameCopyTupleWriterFactory;
    ITreeIndexTupleWriterFactory rtreeLeafFrameBulkLoadWriterFactory;
    if (isPointMBR) {
        int keyFieldCount = rtreeCmpFactories.length;
        int valueFieldCount = btreeComparatorFactories.length - keyFieldCount;
        rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
        rtreeLeafFrameCopyTupleWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
        rtreeLeafFrameBulkLoadWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
    } else {
        rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
        rtreeLeafFrameCopyTupleWriterFactory = new LSMRTreeCopyTupleWriterFactory(typeTraits);
        rtreeLeafFrameBulkLoadWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
    }
    LSMRTreeTupleWriterFactory btreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, true);
    ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeInteriorFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory rtreeLeafFrameBulkLoadFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameBulkLoadWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
    ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameCopyTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, rtreeInteriorFrameFactory, copyTupleLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
    TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameBulkLoadFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
    // The first field is for the sorted curve (e.g. Hilbert curve), and the
    // second field is for the primary key.
    int[] comparatorFields = new int[btreeComparatorFactories.length - rtreeCmpFactories.length + 1];
    IBinaryComparatorFactory[] linearizerArray = new IBinaryComparatorFactory[btreeComparatorFactories.length - rtreeCmpFactories.length + 1];
    comparatorFields[0] = 0;
    for (int i = 1; i < comparatorFields.length; i++) {
        comparatorFields[i] = rtreeCmpFactories.length - 1 + i;
    }
    linearizerArray[0] = linearizerCmpFactory;
    int j = 1;
    for (int i = rtreeCmpFactories.length; i < btreeComparatorFactories.length; i++) {
        linearizerArray[j] = btreeComparatorFactories[i];
        j++;
    }
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    ILSMIndexFileManager fileNameManager = new LSMRTreeWithAntiMatterTuplesFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory);
    return new LSMRTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, filterHelper, filterFrameFactory, filterManager, diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeComparatorFactories, linearizerCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, filterFields, durable, isPointMBR);
}
Also used : LSMRTreeCopyTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeCopyTupleWriterFactory) LSMRTreeTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeTupleWriterFactory) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) LSMTypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) ITreeIndexTupleWriterFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) LSMComponentFilterFrameFactory(org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory) LSMComponentFilterManager(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager) RTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMRTreeWithAntiMatterTuplesFileManager(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuplesFileManager) LSMRTreeTupleWriterFactoryForPointMBR(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeTupleWriterFactoryForPointMBR) LSMRTreeWithAntiMatterTuples(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples) RTreeFactory(org.apache.hyracks.storage.am.lsm.rtree.impls.RTreeFactory) LSMRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree) RTree(org.apache.hyracks.storage.am.rtree.impls.RTree) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) ComponentFilterHelper(org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper) RTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory)

Example 5 with ILSMIndexFileManager

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager in project asterixdb by apache.

the class LSMRTreeUtils method createExternalRTree.

public static ExternalRTree createExternalRTree(IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizeCmpFactory, int[] buddyBTreeFields, boolean durable, boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
    int keyFieldCount = rtreeCmpFactories.length;
    int valueFieldCount = typeTraits.length - keyFieldCount;
    ITypeTraits[] btreeTypeTraits = new ITypeTraits[valueFieldCount];
    for (int i = 0; i < buddyBTreeFields.length; i++) {
        btreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
    }
    ITreeIndexTupleWriterFactory rtreeInteriorFrameTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
    ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
    if (isPointMBR) {
        rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false, false);
    } else {
        rtreeLeafFrameTupleWriterFactory = rtreeInteriorFrameTupleWriterFactory;
    }
    ITreeIndexTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(btreeTypeTraits, true);
    ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeInteriorFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
    ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
    TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeTypeTraits.length);
    int[] comparatorFields = { 0 };
    IBinaryComparatorFactory[] linearizerArray = { linearizeCmpFactory };
    int[] bloomFilterKeyFields = new int[btreeCmpFactories.length];
    for (int i = 0; i < btreeCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory, diskBTreeFactory);
    ExternalRTree lsmTree = new ExternalRTree(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable, isPointMBR);
    return lsmTree;
}
Also used : ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) ITreeIndexTupleWriterFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) RTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMTypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory) LSMRTreeTupleWriterFactoryForPointMBR(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeTupleWriterFactoryForPointMBR) RTreeFactory(org.apache.hyracks.storage.am.lsm.rtree.impls.RTreeFactory) LSMRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree) RTree(org.apache.hyracks.storage.am.rtree.impls.RTree) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) LSMRTreeFileManager(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) RTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory)

Aggregations

ILSMIndexFileManager (org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager)8 BTreeNSMInteriorFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory)6 BTreeNSMLeafFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory)6 ITreeIndexFrameFactory (org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory)6 BloomFilterFactory (org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory)5 BTree (org.apache.hyracks.storage.am.btree.impls.BTree)5 BTreeFactory (org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory)5 IBinaryComparatorFactory (org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory)4 TypeAwareTupleWriterFactory (org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory)4 ITypeTraits (org.apache.hyracks.api.dataflow.value.ITypeTraits)3 ITreeIndexTupleWriterFactory (org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory)3 ExternalBTree (org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree)3 LSMBTree (org.apache.hyracks.storage.am.lsm.btree.impls.LSMBTree)3 LSMBTreeCopyTupleWriterFactory (org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeCopyTupleWriterFactory)3 LSMBTreeTupleWriterFactory (org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeTupleWriterFactory)3 LSMComponentFilterFrameFactory (org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory)3 ComponentFilterHelper (org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper)3 LSMComponentFilterManager (org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager)3 ExternalRTree (org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree)3 LSMRTree (org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree)3