Examples with IBinaryComparatorFactory org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory used on opensource projects

Example 66 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class BTreeUpdateSearchTest method test01.

// Update scan test on fixed-length tuples.
@Test
public void test01() throws Exception {
    IBufferCache bufferCache = harness.getBufferCache();
    // declare fields
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // declare keys
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    @SuppressWarnings("rawtypes") ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    Random rnd = new Random();
    rnd.setSeed(50);
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("INSERTING INTO TREE");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference insertTuple = new ArrayTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    int numInserts = 10000;
    for (int i = 0; i < numInserts; i++) {
        int f0 = rnd.nextInt() % 10000;
        int f1 = 5;
        TupleUtils.createIntegerTuple(tb, insertTuple, f0, f1);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 10000 == 0) {
                long end = System.currentTimeMillis();
                LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
            }
        }
        try {
            indexAccessor.insert(insertTuple);
        } catch (HyracksDataException hde) {
            if (hde.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                hde.printStackTrace();
                throw hde;
            }
        }
    }
    long end = System.currentTimeMillis();
    long duration = end - start;
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("DURATION: " + duration);
    }
    // Update scan.
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("UPDATE SCAN:");
    }
    // Set the cursor to X latch nodes.
    ITreeIndexCursor updateScanCursor = new BTreeRangeSearchCursor(leafFrame, true);
    RangePredicate nullPred = new RangePredicate(null, null, true, true, null, null);
    indexAccessor.search(updateScanCursor, nullPred);
    try {
        while (updateScanCursor.hasNext()) {
            updateScanCursor.next();
            ITupleReference tuple = updateScanCursor.getTuple();
            // Change the value field.
            IntegerPointable.setInteger(tuple.getFieldData(1), tuple.getFieldStart(1), 10);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        updateScanCursor.close();
    }
    // Ordered scan to verify the values.
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("ORDERED SCAN:");
    }
    // Set the cursor to X latch nodes.
    ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame, true);
    indexAccessor.search(scanCursor, nullPred);
    try {
        while (scanCursor.hasNext()) {
            scanCursor.next();
            ITupleReference tuple = scanCursor.getTuple();
            String rec = TupleUtils.printTuple(tuple, recDescSers);
            if (LOGGER.isLoggable(Level.INFO)) {
                LOGGER.info(rec);
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        scanCursor.close();
    }
    btree.deactivate();
    btree.destroy();
}

Example 67 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class PerfExperiment method main.

public static void main(String[] args) throws Exception {
    // Disable logging so we can better see the output times.
    Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
    while (loggers.hasMoreElements()) {
        String loggerName = loggers.nextElement();
        Logger logger = LogManager.getLogManager().getLogger(loggerName);
        logger.setLevel(Level.OFF);
    }
    boolean sorted = Boolean.parseBoolean(args[0]);
    int numThreads = Integer.parseInt(args[1]);
    //int numTuples = 100000; // 100K
    //int numTuples = 1000000; // 1M
    //int numTuples = 2000000; // 2M
    //int numTuples = 3000000; // 3M
    //int numTuples = 10000000; // 10M
    // 20M
    int numTuples = 20000000;
    //int numTuples = 30000000; // 30M
    //int numTuples = 40000000; // 40M
    //int numTuples = 60000000; // 60M
    //int numTuples = 100000000; // 100M
    //int numTuples = 200000000; // 200M
    int batchSize = 10000;
    int numBatches = numTuples / batchSize;
    int payLoadSize = 240;
    ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, payLoadSize);
    IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
    int[] bloomFilterKeyFields = new int[cmpFactories.length];
    for (int i = 0; i < bloomFilterKeyFields.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    double bloomFilterFalsePositiveRate = 0.01;
    //int repeats = 1000;
    int repeats = 1;
    long[] times = new long[repeats];
    //        boolean sorted = true;
    for (int i = 0; i < repeats; i++) {
        //ConcurrentSkipListRunner runner = new ConcurrentSkipListRunner(numBatches, batchSize, tupleSize, typeTraits, cmp);
        //InMemoryBTreeRunner runner = new InMemoryBTreeRunner(numBatches, 8192, 100000, typeTraits, cmpFactories);
        //BTreeBulkLoadRunner runner = new BTreeBulkLoadRunner(numBatches, 8192, 100000, typeTraits, cmp, 1.0f);
        //BTreeRunner runner = new BTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
        //String btreeName = "071211";
        //BTreeSearchRunner runner = new BTreeSearchRunner(btreeName, 10, numBatches, 8192, 25000, typeTraits, cmp);
        //LSMTreeRunner runner = new LSMTreeRunner(numBatches, 8192, 100, 8192, 250, typeTraits, cmp);
        //LSMTreeSearchRunner runner = new LSMTreeSearchRunner(100000, numBatches, 8192, 24750, 8192, 250, typeTraits, cmp);
        // 128kb
        int inMemPageSize = 131072;
        int onDiskPageSize = inMemPageSize;
        // 1GB
        int inMemNumPages = 8192;
        // 2GB
        int onDiskNumPages = 16384;
        LSMTreeRunner runner = new LSMTreeRunner(numBatches, inMemPageSize, inMemNumPages, onDiskPageSize, onDiskNumPages, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate);
        DataGenThread dataGen = new DataGenThread(numThreads, numBatches, batchSize, fieldSerdes, payLoadSize, 50, 10, sorted);
        dataGen.start();
        runner.reset();
        times[i] = runner.runExperiment(dataGen, numThreads);
        System.out.println("TIME " + i + ": " + times[i] + "ms");
        runner.deinit();
    }
    long avgTime = 0;
    for (int i = 0; i < repeats; i++) {
        avgTime += times[i];
    }
    avgTime /= repeats;
    System.out.println("AVG TIME: " + avgTime + "ms");
}

Example 68 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class LSMBTreeTestContext method create.

public static LSMBTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes, int numKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, IMetadataPageManagerFactory metadataPageManagerFactory, boolean filtered) throws Exception {
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
    IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
    int[] bloomFilterKeyFields = new int[numKeyFields];
    for (int i = 0; i < numKeyFields; ++i) {
        bloomFilterKeyFields[i] = i;
    }
    LSMBTree lsmTree;
    if (filtered) {
        ITypeTraits[] filterTypeTraits = new ITypeTraits[1];
        filterTypeTraits[0] = typeTraits[0];
        int[] btreefields = new int[typeTraits.length];
        for (int i = 0; i < btreefields.length; i++) {
            btreefields[i] = i;
        }
        int[] filterfields = { btreefields.length };
        IBinaryComparatorFactory[] filterCmp = { cmpFactories[0] };
        lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, diskFileMapProvider, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, true, filterTypeTraits, filterCmp, btreefields, filterfields, true, metadataPageManagerFactory);
    } else {
        lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, diskFileMapProvider, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, true, null, null, null, null, true, metadataPageManagerFactory);
    }
    LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered);
    return testCtx;
}

Example 69 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class BTreeSearchCursorTest method nonUniqueFieldPrefixIndexTest.

@Test
public void nonUniqueFieldPrefixIndexTest() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE FIELD-PREFIX COMPRESSED INDEX");
    }
    IBufferCache bufferCache = harness.getBufferCache();
    // declare keys
    int keyFieldCount = 2;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
    ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
    IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
    IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
    btree.create();
    btree.activate();
    ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    ITreeIndexAccessor indexAccessor = btree.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // generate keys
    int numKeys = 50;
    int maxKey = 10;
    ArrayList<Integer> keys = new ArrayList<>();
    for (int i = 0; i < numKeys; i++) {
        int k = rnd.nextInt() % maxKey;
        keys.add(k);
    }
    Collections.sort(keys);
    // insert keys into btree
    for (int i = 0; i < keys.size(); i++) {
        TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
        tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
        try {
            indexAccessor.insert(tuple);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    int minSearchKey = -100;
    int maxSearchKey = 100;
    // forward searches
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, false));
    Assert.assertTrue(performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false));
    btree.deactivate();
    btree.destroy();
}

Example 70 with IBinaryComparatorFactory

use of org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory in project asterixdb by apache.

the class BTreePageSizePerf method main.

public static void main(String[] args) throws Exception {
    // Disable logging so we can better see the output times.
    Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
    while (loggers.hasMoreElements()) {
        String loggerName = loggers.nextElement();
        Logger logger = LogManager.getLogManager().getLogger(loggerName);
        logger.setLevel(Level.OFF);
    }
    int numTuples = 1000000;
    int batchSize = 10000;
    int numBatches = numTuples / batchSize;
    ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, 30);
    IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
    runExperiment(numBatches, batchSize, 1024, 100000, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 2048, 100000, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 4096, 25000, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 8192, 12500, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 16384, 6250, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 32768, 3125, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 65536, 1564, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 131072, 782, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 262144, 391, fieldSerdes, cmpFactories, typeTraits);
}