Examples with ExternalBTree org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree used on opensource projects

Example 1 with ExternalBTree

use of org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree in project asterixdb by apache.

the class LSMBTreeUtil method createExternalBTree.

public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
    LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    LSMBTreeTupleWriterFactory deleteTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, true);
    LSMBTreeCopyTupleWriterFactory copyTupleWriterFactory = new LSMBTreeCopyTupleWriterFactory(typeTraits, cmpFactories.length);
    ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
    ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
    ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
    // This is the tuple writer that can do both inserts and deletes
    LSMBTreeTupleWriterFactory transactionTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
    // This is the leaf frame factory for transaction components since it
    // can be used for both inserts and deletes
    ITreeIndexFrameFactory transactionLeafFrameFactory = new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory);
    TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
    TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
    BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
    // This is the component factory for transactions
    TreeIndexFactory<BTree> transactionBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length);
    //TODO remove BloomFilter from external dataset's secondary LSMBTree index
    ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true);
    // the disk only index uses an empty ArrayList for virtual buffer caches
    ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, durable);
    return lsmTree;
}

Example 2 with ExternalBTree

use of org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree in project asterixdb by apache.

the class ExternalFilesIndexModificationOperatorDescriptor method createPushRuntime.

@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
    return new AbstractOperatorNodePushable() {

        @Override
        public void initialize() throws HyracksDataException {
            final IIndexDataflowHelper indexHelper = dataflowHelperFactory.create(ctx, partition);
            FileIndexTupleTranslator filesTupleTranslator = new FileIndexTupleTranslator();
            // Open and get
            indexHelper.open();
            IIndex index = indexHelper.getIndexInstance();
            LSMTwoPCBTreeBulkLoader bulkLoader = null;
            try {
                bulkLoader = (LSMTwoPCBTreeBulkLoader) ((ExternalBTree) index).createTransactionBulkLoader(BTree.DEFAULT_FILL_FACTOR, false, files.size());
                // The files must be ordered according to their numbers
                for (ExternalFile file : files) {
                    switch(file.getPendingOp()) {
                        case ADD_OP:
                        case APPEND_OP:
                            bulkLoader.add(filesTupleTranslator.getTupleFromFile(file));
                            break;
                        case DROP_OP:
                            bulkLoader.delete(filesTupleTranslator.getTupleFromFile(file));
                            break;
                        case NO_OP:
                            break;
                        default:
                            throw HyracksDataException.create(ErrorCode.UNKNOWN_EXTERNAL_FILE_PENDING_OP, file.getPendingOp());
                    }
                }
                bulkLoader.end();
            } catch (Exception e) {
                if (bulkLoader != null) {
                    bulkLoader.abort();
                }
                throw HyracksDataException.create(e);
            } finally {
                indexHelper.close();
            }
        }

        @Override
        public void deinitialize() throws HyracksDataException {
        }

        @Override
        public int getInputArity() {
            return 0;
        }

        @Override
        public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) throws HyracksDataException {
        }

        @Override
        public IFrameWriter getInputFrameWriter(int index) {
            return null;
        }
    };
}