Examples with ArrayTupleBuilder org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder used on opensource projects

Example 91 with ArrayTupleBuilder

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.

the class AbstractRunGeneratorTest method prepareData.

static void prepareData(IHyracksTaskContext ctx, List<IFrame> frameList, int minDataSize, int minRecordSize, int maxRecordSize, Map<Integer, String> specialData, Map<Integer, String> keyValuePair) throws HyracksDataException {
    ArrayTupleBuilder tb = new ArrayTupleBuilder(RecordDesc.getFieldCount());
    FrameTupleAppender appender = new FrameTupleAppender();
    int datasize = 0;
    if (specialData != null) {
        for (Map.Entry<Integer, String> entry : specialData.entrySet()) {
            tb.reset();
            tb.addField(IntegerSerializerDeserializer.INSTANCE, entry.getKey());
            tb.addField(new UTF8StringSerializerDeserializer(), entry.getValue());
            VSizeFrame frame = new VSizeFrame(ctx, FrameHelper.calcAlignedFrameSizeToStore(tb.getFieldEndOffsets().length, tb.getSize(), ctx.getInitialFrameSize()));
            appender.reset(frame, true);
            assertTrue(appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize()));
            frameList.add(frame);
            datasize += frame.getFrameSize();
        }
        keyValuePair.putAll(specialData);
    }
    VSizeFrame frame = new VSizeFrame(ctx, ctx.getInitialFrameSize());
    appender.reset(frame, true);
    while (datasize < minDataSize) {
        tb.reset();
        int key = GRandom.nextInt(minDataSize + 1);
        if (!keyValuePair.containsKey(key)) {
            String value = generateRandomRecord(minRecordSize, maxRecordSize);
            tb.addField(IntegerSerializerDeserializer.INSTANCE, key);
            tb.addField(new UTF8StringSerializerDeserializer(), value);
            if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
                frameList.add(frame);
                datasize += frame.getFrameSize();
                frame = new VSizeFrame(ctx, FrameHelper.calcAlignedFrameSizeToStore(tb.getFieldEndOffsets().length, tb.getSize(), ctx.getInitialFrameSize()));
                appender.reset(frame, true);
                assertTrue(appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize()));
            }
            keyValuePair.put(key, value);
        }
    }
    if (appender.getTupleCount() > 0) {
        frameList.add(frame);
    }
}

Example 92 with ArrayTupleBuilder

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.

the class RunMergingFrameReaderTest method prepareSortedData.

static int prepareSortedData(int minDataSize, int minRecordSize, int maxRecordSize, Map<Integer, String> specialData, Map<Integer, String> result) throws HyracksDataException {
    ArrayTupleBuilder tb = new ArrayTupleBuilder(RecordDesc.getFieldCount());
    int datasize = 0;
    int maxtuple = 0;
    if (specialData != null) {
        for (Map.Entry<Integer, String> entry : specialData.entrySet()) {
            tb.reset();
            tb.addField(IntegerSerializerDeserializer.INSTANCE, entry.getKey());
            tb.addField(new UTF8StringSerializerDeserializer(), entry.getValue());
            int size = tb.getSize() + tb.getFieldEndOffsets().length * 4;
            datasize += size;
            if (size > maxtuple) {
                maxtuple = size;
            }
        }
        result.putAll(specialData);
    }
    while (datasize < minDataSize) {
        String value = generateRandomRecord(minRecordSize, maxRecordSize);
        tb.reset();
        int key = GRandom.nextInt(datasize + 1);
        if (!result.containsKey(key)) {
            tb.addField(IntegerSerializerDeserializer.INSTANCE, key);
            tb.addField(new UTF8StringSerializerDeserializer(), value);
            int size = tb.getSize() + tb.getFieldEndOffsets().length * 4;
            datasize += size;
            if (size > maxtuple) {
                maxtuple = size;
            }
            if (datasize < minDataSize) {
                result.put(key, value);
            }
        }
    }
    return maxtuple;
}

Example 93 with ArrayTupleBuilder

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.

the class TopKRunGeneratorTest method prepareSortedData.

private ByteBuffer prepareSortedData(Map<Integer, String> keyValuePair) throws HyracksDataException {
    ByteBuffer buffer = ByteBuffer.allocate(PAGE_SIZE * NUM_PAGES);
    IFrame inputFrame = new FixedSizeFrame(buffer);
    FrameTupleAppender appender = new FrameTupleAppender();
    appender.reset(inputFrame, true);
    ArrayTupleBuilder builder = new ArrayTupleBuilder(RecordDesc.getFieldCount());
    for (Map.Entry<Integer, String> entry : keyValuePair.entrySet()) {
        builder.reset();
        builder.addField(SerDers[0], entry.getKey());
        builder.addField(SerDers[1], entry.getValue());
        appender.append(builder.getFieldEndOffsets(), builder.getByteArray(), 0, builder.getSize());
    }
    return buffer;
}

Example 94 with ArrayTupleBuilder

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.

the class AbstractRTreeExamplesTest method bulkLoadExample.

/**
     * Bulk load example. Load a tree with 10,000 tuples.
     */
@Test
public void bulkLoadExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Bulk load example");
    }
    // Declare fields.
    int fieldCount = 5;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    typeTraits[3] = IntegerPointable.TYPE_TRAITS;
    typeTraits[4] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // Declare RTree keys.
    int rtreeKeyFieldCount = 4;
    IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
    rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // Declare BTree keys, this will only be used for LSMRTree
    int btreeKeyFieldCount;
    IBinaryComparatorFactory[] btreeCmpFactories;
    int[] btreeFields = null;
    if (rTreeType == RTreeType.LSMRTREE) {
        //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
        btreeKeyFieldCount = 1;
        btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
        btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeFields = new int[btreeKeyFieldCount];
        for (int i = 0; i < btreeKeyFieldCount; i++) {
            btreeFields[i] = rtreeKeyFieldCount + i;
        }
    } else {
        btreeKeyFieldCount = 5;
        btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
        btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    }
    // create value providers
    IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(rtreeCmpFactories.length, IntegerPointable.FACTORY);
    //6
    ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    // Load records.
    int numInserts = 10000;
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Bulk loading " + numInserts + " tuples");
    }
    long start = System.currentTimeMillis();
    IIndexBulkLoader bulkLoader = treeIndex.createBulkLoader(0.7f, false, numInserts, true);
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    for (int i = 0; i < numInserts; i++) {
        int p1x = rnd.nextInt();
        int p1y = rnd.nextInt();
        int p2x = rnd.nextInt();
        int p2y = rnd.nextInt();
        int pk = 5;
        TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), pk);
        bulkLoader.add(tuple);
    }
    bulkLoader.end();
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " tuples loaded in " + (end - start) + "ms");
    }
    IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    // Build key.
    ArrayTupleBuilder keyTb = new ArrayTupleBuilder(rtreeKeyFieldCount);
    ArrayTupleReference key = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
    rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key, null, null);
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 95 with ArrayTupleBuilder

use of org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder in project asterixdb by apache.

the class AbstractRTreeExamplesTest method twoDimensionsExample.

/**
     * Two Dimensions Example. Create an RTree index of two dimensions, where
     * they keys are of type integer, and the payload is two integer values.
     * Fill index with random values using insertions (not bulk load). Perform
     * scans and range search.
     */
@Test
public void twoDimensionsExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Fixed-Length Key,Value Example.");
    }
    // Declare fields.
    int fieldCount = 6;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    typeTraits[3] = IntegerPointable.TYPE_TRAITS;
    typeTraits[4] = IntegerPointable.TYPE_TRAITS;
    typeTraits[5] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // Declare RTree keys.
    int rtreeKeyFieldCount = 4;
    IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
    rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // Declare BTree keys, this will only be used for LSMRTree
    int btreeKeyFieldCount;
    IBinaryComparatorFactory[] btreeCmpFactories;
    int[] btreeFields = null;
    if (rTreeType == RTreeType.LSMRTREE) {
        //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
        btreeKeyFieldCount = 2;
        btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
        btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeFields = new int[btreeKeyFieldCount];
        for (int i = 0; i < btreeKeyFieldCount; i++) {
            btreeFields[i] = rtreeKeyFieldCount + i;
        }
    } else {
        btreeKeyFieldCount = 6;
        btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
        btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[5] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    }
    // create value providers
    IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(rtreeCmpFactories.length, IntegerPointable.FACTORY);
    ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, RTreePolicyType.RTREE, null, btreeFields, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    int numInserts = 10000;
    for (int i = 0; i < numInserts; i++) {
        int p1x = rnd.nextInt();
        int p1y = rnd.nextInt();
        int p2x = rnd.nextInt();
        int p2y = rnd.nextInt();
        int pk1 = 5;
        int pk2 = 10;
        TupleUtils.createIntegerTuple(tb, tuple, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), pk1, pk2);
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
    }
    scan(indexAccessor, fieldSerdes);
    diskOrderScan(indexAccessor, fieldSerdes);
    // Build key.
    ArrayTupleBuilder keyTb = new ArrayTupleBuilder(rtreeKeyFieldCount);
    ArrayTupleReference key = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(keyTb, key, -1000, -1000, 1000, 1000);
    rangeSearch(rtreeCmpFactories, indexAccessor, fieldSerdes, key, null, null);
    treeIndex.deactivate();
    treeIndex.destroy();
}