Examples with ArrayTupleReference org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference used on opensource projects

Example 31 with ArrayTupleReference

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.

the class OrderedIndexExamplesTest method bulkOrderVerificationExample.

/**
     * Bulk load failure example. Repeatedly loads a tree with 1,000 tuples, of
     * which one tuple at each possible position does not conform to the
     * expected order. We expect the bulk load to fail with an exception.
     */
@Test
public void bulkOrderVerificationExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Bulk load order verification example");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // declare keys
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    Random rnd = new Random();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    int ins = 1000;
    for (int i = 1; i < ins; i++) {
        ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
        treeIndex.create();
        treeIndex.activate();
        // Load sorted records, and expect to fail at tuple i.
        IIndexBulkLoader bulkLoader = treeIndex.createBulkLoader(0.7f, true, ins, true);
        for (int j = 0; j < ins; j++) {
            if (j > i) {
                fail("Bulk load failure test unexpectedly succeeded past tuple: " + j);
            }
            int key = j;
            if (j == i) {
                int swapElementCase = Math.abs(rnd.nextInt()) % 2;
                if (swapElementCase == 0) {
                    // Element equal to previous element.
                    key--;
                } else {
                    // Element smaller than previous element.
                    key -= Math.abs(Math.random() % (ins - 1)) + 1;
                }
            }
            TupleUtils.createIntegerTuple(tb, tuple, key, 5);
            try {
                bulkLoader.add(tuple);
            } catch (HyracksDataException e) {
                if (e.getErrorCode() == ErrorCode.UNSORTED_LOAD_INPUT || e.getErrorCode() == ErrorCode.DUPLICATE_KEY || e.getErrorCode() == ErrorCode.DUPLICATE_LOAD_INPUT) {
                    if (j != i) {
                        fail("Unexpected exception: " + e.getMessage());
                    }
                    // Success.
                    break;
                } else {
                    throw e;
                }
            }
        }
        treeIndex.deactivate();
        treeIndex.destroy();
    }
}

Example 32 with ArrayTupleReference

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.

the class OrderedIndexExamplesTest method twoFixedLengthKeysOneFixedLengthValueExample.

/**
     * Composite Key Example (Non-Unique Index). Create a tree index with two
     * fixed-length key fields and one fixed-length value field. Fill index with
     * random values using insertions (not bulk load) Perform scans and range
     * search.
     */
@Test
public void twoFixedLengthKeysOneFixedLengthValueExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Composite Key Test");
    }
    // Declare fields.
    int fieldCount = 3;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // declare keys
    int keyFieldCount = 2;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    bloomFilterKeyFields[1] = 1;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    int numInserts = 10000;
    for (int i = 0; i < 10000; i++) {
        int f0 = rnd.nextInt() % 2000;
        int f1 = rnd.nextInt() % 1000;
        int f2 = 5;
        TupleUtils.createIntegerTuple(tb, tuple, f0, f1, f2);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 1000 == 0) {
                LOGGER.info("Inserting " + i + " : " + f0 + " " + f1 + " " + f2);
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
    }
    orderedScan(indexAccessor, fieldSerdes);
    diskOrderScan(indexAccessor, fieldSerdes);
    // Build low key.
    ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference lowKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -3);
    // Build high key.
    ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference highKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(highKeyTb, highKey, 3);
    // Prefix-Range search in [-3, 3]
    rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 33 with ArrayTupleReference

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.

the class OrderedIndexExamplesTest method updateExample.

/**
     * Update example. Create a BTree with one variable-length key field and one
     * variable-length value field. Fill B-tree with random values using
     * insertions, then update entries one-by-one. Repeat procedure a few times
     * on same BTree.
     */
@Test
public void updateExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Update example");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
    typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    int maxLength = 10;
    int ins = 10000;
    String[] keys = new String[10000];
    for (int i = 0; i < ins; i++) {
        String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
        keys[i] = f0;
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 1000 == 0) {
                LOGGER.info("Inserting " + i);
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    // Print before doing any updates.
    orderedScan(indexAccessor, fieldSerdes);
    int runs = 3;
    for (int run = 0; run < runs; run++) {
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("Update test run: " + (run + 1) + "/" + runs);
            LOGGER.info("Updating BTree");
        }
        for (int i = 0; i < ins; i++) {
            // Generate a new random value for f1.
            String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
            TupleUtils.createTuple(tb, tuple, fieldSerdes, keys[i], f1);
            if (LOGGER.isLoggable(Level.INFO)) {
                if (i % 1000 == 0) {
                    LOGGER.info("Updating " + i);
                }
            }
            indexAccessor.update(tuple);
        }
        // Do another scan after a round of updates.
        orderedScan(indexAccessor, fieldSerdes);
    }
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 34 with ArrayTupleReference

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.

the class OrderedIndexExamplesTest method pageSplitTestExample.

/**
     * This test the btree page split. Originally this test didn't pass since
     * the btree was spliting by cardinality and not size. Thus, we might end up
     * with a situation where there is not enough space to insert the new tuple
     * after the split which will throw an error and the split won't be
     * propagated to upper level; thus, the tree is corrupted. Now, it split
     * page by size. The correct behavior on abnormally large keys/values.
     */
@Test
public void pageSplitTestExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("BTree page split test.");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
    typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    String key = "111";
    String data = "XXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, key, data);
    indexAccessor.insert(tuple);
    key = "222";
    data = "XXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, key, data);
    indexAccessor.insert(tuple);
    key = "333";
    data = "XXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, key, data);
    indexAccessor.insert(tuple);
    key = "444";
    data = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, key, data);
    indexAccessor.insert(tuple);
    key = "555";
    data = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, key, data);
    indexAccessor.insert(tuple);
    key = "666";
    data = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, key, data);
    indexAccessor.insert(tuple);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 35 with ArrayTupleReference

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference in project asterixdb by apache.

the class RTreeSearchCursorTest method rangeSearchTest.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void rangeSearchTest() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("TESTING RANGE SEARCH CURSOR FOR RTREE");
    }
    IBufferCache bufferCache = harness.getBufferCache();
    // Declare fields.
    int fieldCount = 5;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    typeTraits[3] = IntegerPointable.TYPE_TRAITS;
    typeTraits[4] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // Declare keys.
    int keyFieldCount = 4;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    cmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // create value providers
    IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(cmpFactories.length, IntegerPointable.FACTORY);
    RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
    ITreeIndexMetadataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
    ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory, valueProviderFactories, RTreePolicyType.RTREE, false);
    ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory, valueProviderFactories, RTreePolicyType.RTREE, false);
    IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
    IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
    IMetadataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
    RTree rtree = new RTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference(), false);
    rtree.create();
    rtree.activate();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    ITreeIndexAccessor indexAccessor = rtree.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    int numInserts = 10000;
    ArrayList<RTreeCheckTuple> checkTuples = new ArrayList<>();
    for (int i = 0; i < numInserts; i++) {
        int p1x = rnd.nextInt();
        int p1y = rnd.nextInt();
        int p2x = rnd.nextInt();
        int p2y = rnd.nextInt();
        int pk = rnd.nextInt();
        ;
        TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), pk);
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
        RTreeCheckTuple checkTuple = new RTreeCheckTuple(fieldCount, keyFieldCount);
        checkTuple.appendField(Math.min(p1x, p2x));
        checkTuple.appendField(Math.min(p1y, p2y));
        checkTuple.appendField(Math.max(p1x, p2x));
        checkTuple.appendField(Math.max(p1y, p2y));
        checkTuple.appendField(pk);
        checkTuples.add(checkTuple);
    }
    // Build key.
    ArrayTupleBuilder keyTb = new ArrayTupleBuilder(keyFieldCount);
    ArrayTupleReference key = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
    MultiComparator cmp = MultiComparator.create(cmpFactories);
    ITreeIndexCursor searchCursor = new RTreeSearchCursor(interiorFrame, leafFrame);
    SearchPredicate searchPredicate = new SearchPredicate(key, cmp);
    RTreeCheckTuple keyCheck = (RTreeCheckTuple) rTreeTestUtils.createCheckTupleFromTuple(key, fieldSerdes, keyFieldCount);
    HashMultiSet<RTreeCheckTuple> expectedResult = rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
    rTreeTestUtils.getRangeSearchExpectedResults(checkTuples, keyCheck);
    indexAccessor.search(searchCursor, searchPredicate);
    rTreeTestUtils.checkExpectedResults(searchCursor, expectedResult, fieldSerdes, keyFieldCount, null);
    rtree.deactivate();
    rtree.destroy();
}