Examples with ITypeTraits org.apache.hyracks.api.dataflow.value.ITypeTraits used on opensource projects

Example 51 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class FieldPrefixNSMTest method test01.

@Test
public void test01() throws Exception {
    // declare fields
    int fieldCount = 3;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    // declare keys
    int keyFieldCount = 3;
    IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
    cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
    cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
    cmps[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
    MultiComparator cmp = new MultiComparator(cmps);
    // just for printing
    @SuppressWarnings("rawtypes") ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    Random rnd = new Random();
    rnd.setSeed(50);
    IBufferCache bufferCache = harness.getBufferCache();
    IFileMapProvider fileMapProvider = harness.getFileMapProvider();
    bufferCache.createFile(harness.getFileReference());
    int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference());
    bufferCache.openFile(btreeFileId);
    IHyracksTaskContext ctx = harness.getHyracksTaskContext();
    ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), true);
    try {
        ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
        BTreeFieldPrefixNSMLeafFrame frame = new BTreeFieldPrefixNSMLeafFrame(tupleWriter);
        frame.setPage(page);
        frame.initBuffer((byte) 0);
        frame.setMultiComparator(cmp);
        frame.setPrefixTupleCount(0);
        String before = new String();
        String after = new String();
        int compactFreq = 5;
        int compressFreq = 5;
        int smallMax = 10;
        int numRecords = 1000;
        int[][] savedFields = new int[numRecords][3];
        // insert records with random calls to compact and compress
        for (int i = 0; i < numRecords; i++) {
            if (LOGGER.isLoggable(Level.INFO)) {
                if ((i + 1) % 100 == 0) {
                    LOGGER.info("INSERTING " + (i + 1) + " / " + numRecords);
                }
            }
            int a = rnd.nextInt() % smallMax;
            int b = rnd.nextInt() % smallMax;
            int c = i;
            ITupleReference tuple = createTuple(ctx, a, b, c, false);
            try {
                int targetTupleIndex = frame.findInsertTupleIndex(tuple);
                frame.insert(tuple, targetTupleIndex);
            } catch (Exception e) {
                e.printStackTrace();
            }
            savedFields[i][0] = a;
            savedFields[i][1] = b;
            savedFields[i][2] = c;
            if (rnd.nextInt() % compactFreq == 0) {
                before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                frame.compact();
                after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                Assert.assertEquals(before, after);
            }
            if (rnd.nextInt() % compressFreq == 0) {
                before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                frame.compress();
                after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                Assert.assertEquals(before, after);
            }
        }
        // delete records with random calls to compact and compress
        for (int i = 0; i < numRecords; i++) {
            if (LOGGER.isLoggable(Level.INFO)) {
                if ((i + 1) % 100 == 0) {
                    LOGGER.info("DELETING " + (i + 1) + " / " + numRecords);
                }
            }
            ITupleReference tuple = createTuple(ctx, savedFields[i][0], savedFields[i][1], savedFields[i][2], false);
            try {
                int tupleIndex = frame.findDeleteTupleIndex(tuple);
                frame.delete(tuple, tupleIndex);
            } catch (Exception e) {
            }
            if (rnd.nextInt() % compactFreq == 0) {
                before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                frame.compact();
                after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                Assert.assertEquals(before, after);
            }
            if (rnd.nextInt() % compressFreq == 0) {
                before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                frame.compress();
                after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
                Assert.assertEquals(before, after);
            }
        }
    } finally {
        bufferCache.unpin(page);
        bufferCache.closeFile(btreeFileId);
        bufferCache.close();
    }
}

Example 52 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class LSMComponentFilterReferenceTest method test.

@Test
public void test() throws HyracksDataException {
    LSMComponentFilterReference filter = new LSMComponentFilterReference(new TypeAwareTupleWriter(new ITypeTraits[] { IntegerPointable.TYPE_TRAITS }));
    Assert.assertEquals(filter.getLength(), 0);
    Assert.assertFalse(filter.isMaxTupleSet() || filter.isMinTupleSet());
    filter.writeMaxTuple(TupleUtils.createIntegerTuple(false, Integer.MAX_VALUE));
    Assert.assertFalse(filter.isMinTupleSet());
    Assert.assertTrue(filter.isMaxTupleSet());
    Assert.assertTrue(filter.getLength() == 11);
    filter.writeMinTuple(TupleUtils.createIntegerTuple(false, Integer.MIN_VALUE));
    Assert.assertTrue(filter.isMinTupleSet() && filter.isMaxTupleSet());
    Assert.assertTrue(filter.getLength() == 20);
    byte[] serFilter = filter.getByteArray();
    LSMComponentFilterReference deserFilter = new LSMComponentFilterReference(new TypeAwareTupleWriter((new ITypeTraits[] { IntegerPointable.TYPE_TRAITS })));
    deserFilter.set(serFilter, 0, 20);
    Assert.assertTrue(deserFilter.isMaxTupleSet() && deserFilter.isMinTupleSet());
    Assert.assertEquals(TupleUtils.deserializeTuple(deserFilter.getMinTuple(), new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE })[0], Integer.MIN_VALUE);
    Assert.assertEquals(TupleUtils.deserializeTuple(deserFilter.getMaxTuple(), new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE })[0], Integer.MAX_VALUE);
}

Example 53 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class PerfExperiment method main.

public static void main(String[] args) throws Exception {
    // Disable logging so we can better see the output times.
    Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
    while (loggers.hasMoreElements()) {
        String loggerName = loggers.nextElement();
        Logger logger = LogManager.getLogManager().getLogger(loggerName);
        logger.setLevel(Level.OFF);
    }
    boolean sorted = Boolean.parseBoolean(args[0]);
    int numThreads = Integer.parseInt(args[1]);
    //int numTuples = 100000; // 100K
    //int numTuples = 1000000; // 1M
    //int numTuples = 2000000; // 2M
    //int numTuples = 3000000; // 3M
    //int numTuples = 10000000; // 10M
    // 20M
    int numTuples = 20000000;
    //int numTuples = 30000000; // 30M
    //int numTuples = 40000000; // 40M
    //int numTuples = 60000000; // 60M
    //int numTuples = 100000000; // 100M
    //int numTuples = 200000000; // 200M
    int batchSize = 10000;
    int numBatches = numTuples / batchSize;
    int payLoadSize = 240;
    ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, payLoadSize);
    IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
    int[] bloomFilterKeyFields = new int[cmpFactories.length];
    for (int i = 0; i < bloomFilterKeyFields.length; i++) {
        bloomFilterKeyFields[i] = i;
    }
    double bloomFilterFalsePositiveRate = 0.01;
    //int repeats = 1000;
    int repeats = 1;
    long[] times = new long[repeats];
    //        boolean sorted = true;
    for (int i = 0; i < repeats; i++) {
        //ConcurrentSkipListRunner runner = new ConcurrentSkipListRunner(numBatches, batchSize, tupleSize, typeTraits, cmp);
        //InMemoryBTreeRunner runner = new InMemoryBTreeRunner(numBatches, 8192, 100000, typeTraits, cmpFactories);
        //BTreeBulkLoadRunner runner = new BTreeBulkLoadRunner(numBatches, 8192, 100000, typeTraits, cmp, 1.0f);
        //BTreeRunner runner = new BTreeRunner(numBatches, 8192, 100000, typeTraits, cmp);
        //String btreeName = "071211";
        //BTreeSearchRunner runner = new BTreeSearchRunner(btreeName, 10, numBatches, 8192, 25000, typeTraits, cmp);
        //LSMTreeRunner runner = new LSMTreeRunner(numBatches, 8192, 100, 8192, 250, typeTraits, cmp);
        //LSMTreeSearchRunner runner = new LSMTreeSearchRunner(100000, numBatches, 8192, 24750, 8192, 250, typeTraits, cmp);
        // 128kb
        int inMemPageSize = 131072;
        int onDiskPageSize = inMemPageSize;
        // 1GB
        int inMemNumPages = 8192;
        // 2GB
        int onDiskNumPages = 16384;
        LSMTreeRunner runner = new LSMTreeRunner(numBatches, inMemPageSize, inMemNumPages, onDiskPageSize, onDiskNumPages, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate);
        DataGenThread dataGen = new DataGenThread(numThreads, numBatches, batchSize, fieldSerdes, payLoadSize, 50, 10, sorted);
        dataGen.start();
        runner.reset();
        times[i] = runner.runExperiment(dataGen, numThreads);
        System.out.println("TIME " + i + ": " + times[i] + "ms");
        runner.deinit();
    }
    long avgTime = 0;
    for (int i = 0; i < repeats; i++) {
        avgTime += times[i];
    }
    avgTime /= repeats;
    System.out.println("AVG TIME: " + avgTime + "ms");
}

Example 54 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class LSMBTreeTestContext method create.

public static LSMBTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches, FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ISerializerDeserializer[] fieldSerdes, int numKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, IMetadataPageManagerFactory metadataPageManagerFactory, boolean filtered) throws Exception {
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
    IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
    int[] bloomFilterKeyFields = new int[numKeyFields];
    for (int i = 0; i < numKeyFields; ++i) {
        bloomFilterKeyFields[i] = i;
    }
    LSMBTree lsmTree;
    if (filtered) {
        ITypeTraits[] filterTypeTraits = new ITypeTraits[1];
        filterTypeTraits[0] = typeTraits[0];
        int[] btreefields = new int[typeTraits.length];
        for (int i = 0; i < btreefields.length; i++) {
            btreefields[i] = i;
        }
        int[] filterfields = { btreefields.length };
        IBinaryComparatorFactory[] filterCmp = { cmpFactories[0] };
        lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, diskFileMapProvider, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, true, filterTypeTraits, filterCmp, btreefields, filterfields, true, metadataPageManagerFactory);
    } else {
        lsmTree = LSMBTreeUtil.createLSMTree(ioManager, virtualBufferCaches, file, diskBufferCache, diskFileMapProvider, typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, true, null, null, null, null, true, metadataPageManagerFactory);
    }
    LSMBTreeTestContext testCtx = new LSMBTreeTestContext(fieldSerdes, lsmTree, filtered);
    return testCtx;
}

Example 55 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class BTreePageSizePerf method main.

public static void main(String[] args) throws Exception {
    // Disable logging so we can better see the output times.
    Enumeration<String> loggers = LogManager.getLogManager().getLoggerNames();
    while (loggers.hasMoreElements()) {
        String loggerName = loggers.nextElement();
        Logger logger = LogManager.getLogManager().getLogger(loggerName);
        logger.setLevel(Level.OFF);
    }
    int numTuples = 1000000;
    int batchSize = 10000;
    int numBatches = numTuples / batchSize;
    ISerializerDeserializer[] fieldSerdes = new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE };
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, 30);
    IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
    runExperiment(numBatches, batchSize, 1024, 100000, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 2048, 100000, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 4096, 25000, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 8192, 12500, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 16384, 6250, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 32768, 3125, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 65536, 1564, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 131072, 782, fieldSerdes, cmpFactories, typeTraits);
    runExperiment(numBatches, batchSize, 262144, 391, fieldSerdes, cmpFactories, typeTraits);
}