Examples with ISerializerDeserializer org.apache.hyracks.api.dataflow.value.ISerializerDeserializer used on opensource projects

Example 61 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class OrderedIndexExamplesTest method varLenKeyValueExample.

/**
     * Variable-Length Example. Create a BTree with one variable-length key
     * field and one variable-length value field. Fill BTree with random values
     * using insertions (not bulk load) Perform ordered scans and range search.
     */
@Test
public void varLenKeyValueExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Variable-Length Key,Value Example");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
    typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // Max string length to be generated.
    int maxLength = 10;
    int numInserts = 10000;
    for (int i = 0; i < 10000; i++) {
        String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 1000 == 0) {
                LOGGER.info("Inserting[" + i + "] " + f0 + " " + f1);
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
    }
    orderedScan(indexAccessor, fieldSerdes);
    diskOrderScan(indexAccessor, fieldSerdes);
    // Build low key.
    ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference lowKey = new ArrayTupleReference();
    TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
    // Build high key.
    ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference highKey = new ArrayTupleReference();
    TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
    rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 62 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class DatetimeFromUnixTimeInSecsDescriptor method createEvaluatorFactory.

/* (non-Javadoc)
     * @see org.apache.asterix.runtime.base.IScalarFunctionDynamicDescriptor#createEvaluatorFactory(org.apache.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory[])
     */
@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
            return new IScalarEvaluator() {

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                private IPointable argPtr = new VoidPointable();

                private IScalarEvaluator eval = args[0].createScalarEvaluator(ctx);

                // possible output types
                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<ADateTime> datetimeSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADATETIME);

                private AMutableDateTime aDatetime = new AMutableDateTime(0);

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    eval.evaluate(tuple, argPtr);
                    byte[] bytes = argPtr.getByteArray();
                    int offset = argPtr.getStartOffset();
                    ATypeTag argPtrTypeTag = ATypeTag.VALUE_TYPE_MAPPING[bytes[offset]];
                    switch(argPtrTypeTag) {
                        case TINYINT:
                            aDatetime.setValue(AInt8SerializerDeserializer.getByte(bytes, offset + 1) * 1000l);
                            break;
                        case SMALLINT:
                            aDatetime.setValue(AInt16SerializerDeserializer.getShort(bytes, offset + 1) * 1000l);
                            break;
                        case INTEGER:
                            aDatetime.setValue(AInt32SerializerDeserializer.getInt(bytes, offset + 1) * 1000l);
                            break;
                        case BIGINT:
                            aDatetime.setValue(AInt64SerializerDeserializer.getLong(bytes, offset + 1) * 1000l);
                            break;
                        default:
                            throw new TypeMismatchException(getIdentifier(), 0, bytes[offset], ATypeTag.SERIALIZED_INT8_TYPE_TAG, ATypeTag.SERIALIZED_INT16_TYPE_TAG, ATypeTag.SERIALIZED_INT32_TYPE_TAG, ATypeTag.SERIALIZED_INT64_TYPE_TAG);
                    }
                    datetimeSerde.serialize(aDatetime, out);
                    result.set(resultStorage);
                }
            };
        }
    };
}

Example 63 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class DayOfWeekDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
            return new IScalarEvaluator() {

                private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

                private DataOutput out = resultStorage.getDataOutput();

                private IPointable argPtr = new VoidPointable();

                private IScalarEvaluator eval = args[0].createScalarEvaluator(ctx);

                // possible returning types
                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<AInt64> int64Serde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);

                private AMutableInt64 aInt64 = new AMutableInt64(0);

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    eval.evaluate(tuple, argPtr);
                    byte[] bytes = argPtr.getByteArray();
                    int offset = argPtr.getStartOffset();
                    int daysSinceAnchor;
                    int reminder = 0;
                    if (bytes[offset] == ATypeTag.SERIALIZED_DATETIME_TYPE_TAG) {
                        daysSinceAnchor = (int) (ADateTimeSerializerDeserializer.getChronon(bytes, offset + 1) / GregorianCalendarSystem.CHRONON_OF_DAY);
                        reminder = (int) (ADateTimeSerializerDeserializer.getChronon(bytes, offset + 1) % GregorianCalendarSystem.CHRONON_OF_DAY);
                    } else if (bytes[offset] == ATypeTag.SERIALIZED_DATE_TYPE_TAG) {
                        daysSinceAnchor = ADateSerializerDeserializer.getChronon(bytes, offset + 1);
                    } else {
                        throw new TypeMismatchException(getIdentifier(), 0, bytes[offset], ATypeTag.SERIALIZED_DATETIME_TYPE_TAG, ATypeTag.SERIALIZED_DATE_TYPE_TAG);
                    }
                    // adjust the day before 1970-01-01
                    if (daysSinceAnchor < 0 && reminder != 0) {
                        daysSinceAnchor -= 1;
                    }
                    // compute the weekday (0-based, and 0 = Sunday). Adjustment is needed as
                    // the anchor day is Thursday.
                    int weekday = (daysSinceAnchor + ANCHOR_WEEKDAY) % 7;
                    // handle the negative weekday
                    if (weekday < 0) {
                        weekday += 7;
                    }
                    // convert from 0-based to 1-based (so 7 = Sunday)
                    if (weekday == 0) {
                        weekday = 7;
                    }
                    aInt64.setValue(weekday);
                    int64Serde.serialize(aInt64, out);
                    result.set(resultStorage);
                }
            };
        }
    };
}

Example 64 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class AbstractRTreeExamplesTest method rStarTreePageSplitTestExample.

/**
     * This test the r*tree page split. Originally this test didn't pass since
     * the r*tree assumes always that there will be enough space for the new
     * tuple after split. Now it passes since if there is not space in the
     * designated page, then we will just insert it in the other split page.
     */
@Test
public void rStarTreePageSplitTestExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("R*Tree page split test.");
    }
    // Declare fields.
    int fieldCount = 5;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    typeTraits[2] = IntegerPointable.TYPE_TRAITS;
    typeTraits[3] = IntegerPointable.TYPE_TRAITS;
    typeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
    // Declare RTree keys.
    int rtreeKeyFieldCount = 4;
    IBinaryComparatorFactory[] rtreeCmpFactories = new IBinaryComparatorFactory[rtreeKeyFieldCount];
    rtreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    rtreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // Declare BTree keys, this will only be used for LSMRTree
    int btreeKeyFieldCount;
    IBinaryComparatorFactory[] btreeCmpFactories;
    int[] btreeFields = null;
    if (rTreeType == RTreeType.LSMRTREE) {
        //Parameters look different for LSM RTREE from LSM RTREE WITH ANTI MATTER TUPLES
        btreeKeyFieldCount = 1;
        btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
        btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
        btreeFields = new int[btreeKeyFieldCount];
        for (int i = 0; i < btreeKeyFieldCount; i++) {
            btreeFields[i] = rtreeKeyFieldCount + i;
        }
    } else {
        btreeKeyFieldCount = 5;
        btreeCmpFactories = new IBinaryComparatorFactory[btreeKeyFieldCount];
        btreeCmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[3] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
        btreeCmpFactories[4] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    }
    // create value providers
    IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(rtreeCmpFactories.length, IntegerPointable.FACTORY);
    ITreeIndex treeIndex = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, RTreePolicyType.RSTARTREE, null, btreeFields, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    int p1x = rnd.nextInt();
    int p1y = rnd.nextInt();
    int p2x = rnd.nextInt();
    int p2y = rnd.nextInt();
    String data = "";
    for (int i = 0; i < 210; i++) {
        data += "X";
    }
    TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
    indexAccessor.insert(tuple);
    p1x = rnd.nextInt();
    p1y = rnd.nextInt();
    p2x = rnd.nextInt();
    p2y = rnd.nextInt();
    data = "";
    for (int i = 0; i < 210; i++) {
        data += "X";
    }
    TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
    indexAccessor.insert(tuple);
    p1x = rnd.nextInt();
    p1y = rnd.nextInt();
    p2x = rnd.nextInt();
    p2y = rnd.nextInt();
    data = "XXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
    indexAccessor.insert(tuple);
    p1x = rnd.nextInt();
    p1y = rnd.nextInt();
    p2x = rnd.nextInt();
    p2y = rnd.nextInt();
    data = "XXX";
    TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
    indexAccessor.insert(tuple);
    p1x = rnd.nextInt();
    p1y = rnd.nextInt();
    p2x = rnd.nextInt();
    p2y = rnd.nextInt();
    data = "";
    for (int i = 0; i < 210; i++) {
        data += "X";
    }
    TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
    indexAccessor.insert(tuple);
    p1x = rnd.nextInt();
    p1y = rnd.nextInt();
    p2x = rnd.nextInt();
    p2y = rnd.nextInt();
    data = "";
    for (int i = 0; i < 210; i++) {
        data += "X";
    }
    TupleUtils.createTuple(tb, tuple, fieldSerdes, Math.min(p1x, p2x), Math.min(p1y, p2y), Math.max(p1x, p2x), Math.max(p1y, p2y), data);
    indexAccessor.insert(tuple);
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 65 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class AbstractRTreeMultiThreadTest method runTest.

protected void runTest(ISerializerDeserializer[] fieldSerdes, IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType, int numThreads, TestWorkloadConf conf, String dataMsg) throws HyracksDataException, InterruptedException {
    setUp();
    if (LOGGER.isLoggable(Level.INFO)) {
        String indexTypeName = getIndexTypeName();
        LOGGER.info(indexTypeName + " MultiThread Test:\nData: " + dataMsg + "; Threads: " + numThreads + "; Workload: " + conf.toString() + ".");
    }
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
    IBinaryComparatorFactory[] rtreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeys);
    int[] btreeFields = null;
    IBinaryComparatorFactory[] btreeCmpFactories;
    if (rTreeType == RTreeType.LSMRTREE) {
        int numBtreeFields = fieldSerdes.length - numKeys;
        ISerializerDeserializer[] btreeFieldSerdes = new ISerializerDeserializer[numBtreeFields];
        btreeFields = new int[numBtreeFields];
        for (int i = 0; i < numBtreeFields; i++) {
            btreeFields[i] = numKeys + i;
            btreeFieldSerdes[i] = fieldSerdes[numKeys + i];
        }
        btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(btreeFieldSerdes, numBtreeFields);
    } else {
        btreeCmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, fieldSerdes.length);
    }
    ITreeIndex index = createTreeIndex(typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, btreeFields);
    IIndexTestWorkerFactory workerFactory = getWorkerFactory();
    // 4 batches per thread.
    int batchSize = (NUM_OPERATIONS / numThreads) / 4;
    IndexMultiThreadTestDriver driver = new IndexMultiThreadTestDriver(index, workerFactory, fieldSerdes, conf.ops, conf.opProbs);
    driver.init();
    long[] times = driver.run(numThreads, 1, NUM_OPERATIONS, batchSize);
    driver.deinit();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("RTree MultiThread Test Time: " + times[0] + "ms");
    }
    tearDown();
}