Search in sources :

Example 6 with BTreeNSMLeafFrameFactory

use of org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory in project asterixdb by apache.

the class LSMRTreeUtils method createLSMTreeWithAntiMatterTuples.

public static LSMRTreeWithAntiMatterTuples createLSMTreeWithAntiMatterTuples(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeComparatorFactories, IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizerCmpFactory, int[] rtreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
    ITreeIndexTupleWriterFactory rtreeInteriorFrameTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
    ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory;
    ITreeIndexTupleWriterFactory rtreeLeafFrameCopyTupleWriterFactory;
    ITreeIndexTupleWriterFactory rtreeLeafFrameBulkLoadWriterFactory;
    if (isPointMBR) {
        int keyFieldCount = rtreeCmpFactories.length;
        int valueFieldCount = btreeComparatorFactories.length - keyFieldCount;
        rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
        rtreeLeafFrameCopyTupleWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
        rtreeLeafFrameBulkLoadWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, true, false);
    } else {
        rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
        rtreeLeafFrameCopyTupleWriterFactory = new LSMRTreeCopyTupleWriterFactory(typeTraits);
        rtreeLeafFrameBulkLoadWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, false);
    }
    LSMRTreeTupleWriterFactory btreeTupleWriterFactory = new LSMRTreeTupleWriterFactory(typeTraits, true);
    ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeInteriorFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory rtreeLeafFrameBulkLoadFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameBulkLoadWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
    ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameCopyTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, rtreeInteriorFrameFactory, copyTupleLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
    TreeIndexFactory<RTree> bulkLoadRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameBulkLoadFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
    // The first field is for the sorted curve (e.g. Hilbert curve), and the
    // second field is for the primary key.
    int[] comparatorFields = new int[btreeComparatorFactories.length - rtreeCmpFactories.length + 1];
    IBinaryComparatorFactory[] linearizerArray = new IBinaryComparatorFactory[btreeComparatorFactories.length - rtreeCmpFactories.length + 1];
    comparatorFields[0] = 0;
    for (int i = 1; i < comparatorFields.length; i++) {
        comparatorFields[i] = rtreeCmpFactories.length - 1 + i;
    }
    linearizerArray[0] = linearizerCmpFactory;
    int j = 1;
    for (int i = rtreeCmpFactories.length; i < btreeComparatorFactories.length; i++) {
        linearizerArray[j] = btreeComparatorFactories[i];
        j++;
    }
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    ILSMIndexFileManager fileNameManager = new LSMRTreeWithAntiMatterTuplesFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory);
    return new LSMRTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, bulkLoadRTreeFactory, filterHelper, filterFrameFactory, filterManager, diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeComparatorFactories, linearizerCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, filterFields, durable, isPointMBR);
}
Also used : LSMRTreeCopyTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeCopyTupleWriterFactory) LSMRTreeTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeTupleWriterFactory) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) LSMTypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) ITreeIndexTupleWriterFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) LSMComponentFilterFrameFactory(org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory) LSMComponentFilterManager(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager) RTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMRTreeWithAntiMatterTuplesFileManager(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuplesFileManager) LSMRTreeTupleWriterFactoryForPointMBR(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeTupleWriterFactoryForPointMBR) LSMRTreeWithAntiMatterTuples(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples) RTreeFactory(org.apache.hyracks.storage.am.lsm.rtree.impls.RTreeFactory) LSMRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree) RTree(org.apache.hyracks.storage.am.rtree.impls.RTree) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) ComponentFilterHelper(org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper) RTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory)

Example 7 with BTreeNSMLeafFrameFactory

use of org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory in project asterixdb by apache.

the class LSMRTreeUtils method createExternalRTree.

public static ExternalRTree createExternalRTree(IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizeCmpFactory, int[] buddyBTreeFields, boolean durable, boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
    int keyFieldCount = rtreeCmpFactories.length;
    int valueFieldCount = typeTraits.length - keyFieldCount;
    ITypeTraits[] btreeTypeTraits = new ITypeTraits[valueFieldCount];
    for (int i = 0; i < buddyBTreeFields.length; i++) {
        btreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
    }
    ITreeIndexTupleWriterFactory rtreeInteriorFrameTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
    ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
    if (isPointMBR) {
        rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false, false);
    } else {
        rtreeLeafFrameTupleWriterFactory = rtreeInteriorFrameTupleWriterFactory;
    }
    ITreeIndexTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(btreeTypeTraits, true);
    ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeInteriorFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
    ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
    TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeTypeTraits.length);
    int[] comparatorFields = { 0 };
    IBinaryComparatorFactory[] linearizerArray = { linearizeCmpFactory };
    int[] bloomFilterKeyFields = new int[btreeCmpFactories.length];
    for (int i = 0; i < btreeCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory, diskBTreeFactory);
    ExternalRTree lsmTree = new ExternalRTree(ioManager, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, buddyBTreeFields, durable, isPointMBR);
    return lsmTree;
}
Also used : ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) ITreeIndexTupleWriterFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) RTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMTypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory) LSMRTreeTupleWriterFactoryForPointMBR(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeTupleWriterFactoryForPointMBR) RTreeFactory(org.apache.hyracks.storage.am.lsm.rtree.impls.RTreeFactory) LSMRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree) RTree(org.apache.hyracks.storage.am.rtree.impls.RTree) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) LSMRTreeFileManager(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) RTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory)

Example 8 with BTreeNSMLeafFrameFactory

use of org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory in project asterixdb by apache.

the class LSMRTreeUtils method createLSMTree.

public static LSMRTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories, IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, int[] buddyBTreeFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable, boolean isPointMBR, IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
    int valueFieldCount = buddyBTreeFields.length;
    int keyFieldCount = typeTraits.length - valueFieldCount;
    ITypeTraits[] btreeTypeTraits = new ITypeTraits[valueFieldCount];
    for (int i = 0; i < valueFieldCount; i++) {
        btreeTypeTraits[i] = typeTraits[buddyBTreeFields[i]];
    }
    ITreeIndexTupleWriterFactory rtreeInteriorFrameTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(typeTraits, false);
    ITreeIndexTupleWriterFactory rtreeLeafFrameTupleWriterFactory = null;
    if (isPointMBR) {
        rtreeLeafFrameTupleWriterFactory = new LSMRTreeTupleWriterFactoryForPointMBR(typeTraits, keyFieldCount, valueFieldCount, false, false);
    } else {
        rtreeLeafFrameTupleWriterFactory = rtreeInteriorFrameTupleWriterFactory;
    }
    ITreeIndexTupleWriterFactory btreeTupleWriterFactory = new LSMTypeAwareTupleWriterFactory(btreeTypeTraits, true);
    ITreeIndexFrameFactory rtreeInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(rtreeInteriorFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory rtreeLeafFrameFactory = new RTreeNSMLeafFrameFactory(rtreeLeafFrameTupleWriterFactory, valueProviderFactories, rtreePolicyType, isPointMBR);
    ITreeIndexFrameFactory btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(btreeTupleWriterFactory);
    ITreeIndexFrameFactory btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(btreeTupleWriterFactory);
    TreeIndexFactory<RTree> diskRTreeFactory = new RTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, rtreeCmpFactories, typeTraits.length, isPointMBR);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, btreeCmpFactories, btreeTypeTraits.length);
    int[] comparatorFields = { 0 };
    IBinaryComparatorFactory[] linearizerArray = { linearizeCmpFactory };
    int[] bloomFilterKeyFields = new int[btreeCmpFactories.length];
    for (int i = 0; i < btreeCmpFactories.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    ComponentFilterHelper filterHelper = null;
    LSMComponentFilterFrameFactory filterFrameFactory = null;
    LSMComponentFilterManager filterManager = null;
    if (filterCmpFactories != null) {
        TypeAwareTupleWriterFactory filterTupleWriterFactory = new TypeAwareTupleWriterFactory(filterTypeTraits);
        filterHelper = new ComponentFilterHelper(filterTupleWriterFactory, filterCmpFactories);
        filterFrameFactory = new LSMComponentFilterFrameFactory(filterTupleWriterFactory);
        filterManager = new LSMComponentFilterManager(filterFrameFactory);
    }
    ILSMIndexFileManager fileNameManager = new LSMRTreeFileManager(ioManager, diskFileMapProvider, file, diskRTreeFactory, diskBTreeFactory);
    return new LSMRTree(ioManager, virtualBufferCaches, rtreeInteriorFrameFactory, rtreeLeafFrameFactory, btreeInteriorFrameFactory, btreeLeafFrameFactory, fileNameManager, diskRTreeFactory, diskBTreeFactory, bloomFilterFactory, filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, rtreeCmpFactories, btreeCmpFactories, linearizeCmpFactory, comparatorFields, linearizerArray, mergePolicy, opTracker, ioScheduler, ioOpCallback, rtreeFields, buddyBTreeFields, filterFields, durable, isPointMBR);
}
Also used : ILSMIndexFileManager(org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) LSMComponentFilterFrameFactory(org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterFrameFactory) LSMComponentFilterManager(org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) LSMTypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory) LSMRTreeTupleWriterFactoryForPointMBR(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMRTreeTupleWriterFactoryForPointMBR) LSMRTreeFileManager(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeFileManager) RTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) LSMRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree) LSMTypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.lsm.rtree.tuples.LSMTypeAwareTupleWriterFactory) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) ITreeIndexTupleWriterFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) RTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory) BTreeFactory(org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory) RTreeFactory(org.apache.hyracks.storage.am.lsm.rtree.impls.RTreeFactory) LSMRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree) RTree(org.apache.hyracks.storage.am.rtree.impls.RTree) ExternalRTree(org.apache.hyracks.storage.am.lsm.rtree.impls.ExternalRTree) BloomFilterFactory(org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory) ComponentFilterHelper(org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper)

Example 9 with BTreeNSMLeafFrameFactory

use of org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory in project asterixdb by apache.

the class BTreeStatsTest method test01.

@Test
public void test01() throws Exception {
    TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
    IBufferCache bufferCache = harness.getBufferCache();
    IFileMapProvider fmp = harness.getFileMapProvider();
    // declare fields
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // declare keys
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
    ITreeIndexMetadataFrame metaFrame = metaFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    Random rnd = new Random();
    rnd.setSeed(50);
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("INSERTING INTO TREE");
    }
    IFrame frame = new VSizeFrame(ctx);
    FrameTupleAppender appender = new FrameTupleAppender();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    DataOutput dos = tb.getDataOutput();
    ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
    IFrameTupleAccessor accessor = new FrameTupleAccessor(recDesc);
    accessor.reset(frame.getBuffer());
    FrameTupleReference tuple = new FrameTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // 10000
    for (int i = 0; i < 100000; i++) {
        int f0 = rnd.nextInt() % 100000;
        int f1 = 5;
        tb.reset();
        IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
        tb.addFieldEndOffset();
        IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
        tb.addFieldEndOffset();
        appender.reset(frame, true);
        appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
        tuple.reset(accessor, 0);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 10000 == 0) {
                long end = System.currentTimeMillis();
                LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                e.printStackTrace();
                throw e;
            }
        }
    }
    int fileId = fmp.lookupFileId(harness.getFileReference());
    TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId, btree.getRootPageId());
    TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("\n" + stats.toString());
    }
    TreeIndexBufferCacheWarmup bufferCacheWarmup = new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, fileId);
    bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 });
    btree.deactivate();
    btree.destroy();
    bufferCache.close();
}
Also used : DataOutput(java.io.DataOutput) IBTreeInteriorFrame(org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame) IFrame(org.apache.hyracks.api.comm.IFrame) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) TreeIndexStatsGatherer(org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) ITreeIndexMetadataFrame(org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) ITreeIndexMetadataFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrameFactory) LIFOMetaDataFrameFactory(org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory) Random(java.util.Random) IBTreeLeafFrame(org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame) FrameTupleAppender(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender) IFrameTupleAccessor(org.apache.hyracks.api.comm.IFrameTupleAccessor) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) FrameTupleReference(org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference) ArrayTupleBuilder(org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder) TreeIndexStats(org.apache.hyracks.storage.am.common.util.TreeIndexStats) IMetadataPageManager(org.apache.hyracks.storage.am.common.api.IMetadataPageManager) LinkedMetaDataPageManager(org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager) VSizeFrame(org.apache.hyracks.api.comm.VSizeFrame) ISerializerDeserializer(org.apache.hyracks.api.dataflow.value.ISerializerDeserializer) ITreeIndexAccessor(org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) IFileMapProvider(org.apache.hyracks.storage.common.file.IFileMapProvider) TreeIndexBufferCacheWarmup(org.apache.hyracks.storage.am.common.util.TreeIndexBufferCacheWarmup) IBufferCache(org.apache.hyracks.storage.common.buffercache.IBufferCache) FrameTupleAccessor(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor) IFrameTupleAccessor(org.apache.hyracks.api.comm.IFrameTupleAccessor) Test(org.junit.Test) AbstractBTreeTest(org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest)

Example 10 with BTreeNSMLeafFrameFactory

use of org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory in project asterixdb by apache.

the class BTreeUpdateSearchTest method test01.

// Update scan test on fixed-length tuples.
@Test
public void test01() throws Exception {
    IBufferCache bufferCache = harness.getBufferCache();
    // declare fields
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // declare keys
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    @SuppressWarnings("rawtypes") ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    Random rnd = new Random();
    rnd.setSeed(50);
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("INSERTING INTO TREE");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference insertTuple = new ArrayTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    int numInserts = 10000;
    for (int i = 0; i < numInserts; i++) {
        int f0 = rnd.nextInt() % 10000;
        int f1 = 5;
        TupleUtils.createIntegerTuple(tb, insertTuple, f0, f1);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 10000 == 0) {
                long end = System.currentTimeMillis();
                LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
            }
        }
        try {
            indexAccessor.insert(insertTuple);
        } catch (HyracksDataException hde) {
            if (hde.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                hde.printStackTrace();
                throw hde;
            }
        }
    }
    long end = System.currentTimeMillis();
    long duration = end - start;
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("DURATION: " + duration);
    }
    // Update scan.
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("UPDATE SCAN:");
    }
    // Set the cursor to X latch nodes.
    ITreeIndexCursor updateScanCursor = new BTreeRangeSearchCursor(leafFrame, true);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    indexAccessor.search(updateScanCursor, nullPred);
    try {
        while (updateScanCursor.hasNext()) {
            updateScanCursor.next();
            ITupleReference tuple = updateScanCursor.getTuple();
            // Change the value field.
            IntegerPointable.setInteger(tuple.getFieldData(1), tuple.getFieldStart(1), 10);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        updateScanCursor.close();
    }
    // Ordered scan to verify the values.
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("ORDERED SCAN:");
    }
    // Set the cursor to X latch nodes.
    ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame, true);
    indexAccessor.search(scanCursor, nullPred);
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            ITupleReference tuple = scanCursor.getTuple();
            String rec = TupleUtils.printTuple(tuple, recDescSers);
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info(rec);
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        scanCursor.close();
    }
    btree.deactivate();
    btree.destroy();
}
Also used : RangePredicate(org.apache.hyracks.storage.am.btree.impls.RangePredicate) BTreeRangeSearchCursor(org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor) ArrayTupleReference(org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference) BTree(org.apache.hyracks.storage.am.btree.impls.BTree) BTreeNSMInteriorFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory) ITreeIndexFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory) BTreeNSMLeafFrameFactory(org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory) ITreeIndexMetadataFrameFactory(org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrameFactory) LIFOMetaDataFrameFactory(org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory) Random(java.util.Random) IBTreeLeafFrame(org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame) ITreeIndexCursor(org.apache.hyracks.storage.am.common.api.ITreeIndexCursor) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) TypeAwareTupleWriterFactory(org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) ArrayTupleBuilder(org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder) IMetadataPageManager(org.apache.hyracks.storage.am.common.api.IMetadataPageManager) ISerializerDeserializer(org.apache.hyracks.api.dataflow.value.ISerializerDeserializer) LinkedMetaDataPageManager(org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager) ITreeIndexAccessor(org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) ITupleReference(org.apache.hyracks.dataflow.common.data.accessors.ITupleReference) IBufferCache(org.apache.hyracks.storage.common.buffercache.IBufferCache) Test(org.junit.Test) AbstractBTreeTest(org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest)

Aggregations

BTreeNSMInteriorFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory)12 BTreeNSMLeafFrameFactory (org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory)12 ITreeIndexFrameFactory (org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory)12 BTree (org.apache.hyracks.storage.am.btree.impls.BTree)11 IBinaryComparatorFactory (org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory)9 TypeAwareTupleWriterFactory (org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory)7 ILSMIndexFileManager (org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager)6 ITypeTraits (org.apache.hyracks.api.dataflow.value.ITypeTraits)5 HyracksDataException (org.apache.hyracks.api.exceptions.HyracksDataException)5 ArrayTupleBuilder (org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder)5 BloomFilterFactory (org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory)5 IBTreeLeafFrame (org.apache.hyracks.storage.am.btree.api.IBTreeLeafFrame)5 AbstractBTreeTest (org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest)5 IMetadataPageManager (org.apache.hyracks.storage.am.common.api.IMetadataPageManager)5 ITreeIndexAccessor (org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor)5 LinkedMetaDataPageManager (org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager)5 BTreeFactory (org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory)5 IBufferCache (org.apache.hyracks.storage.common.buffercache.IBufferCache)5 Test (org.junit.Test)5 ArrayTupleReference (org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference)4