Examples with ISerializerDeserializer org.apache.hyracks.api.dataflow.value.ISerializerDeserializer used on opensource projects

Example 11 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class OrderedIndexExamplesTest method fixedLengthKeyValueExample.

/**
     * Fixed-Length Key,Value Example. Create a tree index with one fixed-length
     * key field and one fixed-length value field. Fill index with random values
     * using insertions (not bulk load). Perform scans and range search.
     */
@Test
public void fixedLengthKeyValueExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Fixed-Length Key,Value Example.");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    int numInserts = 10000;
    for (int i = 0; i < numInserts; i++) {
        int f0 = rnd.nextInt() % numInserts;
        int f1 = 5;
        TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 1000 == 0) {
                LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
    }
    orderedScan(indexAccessor, fieldSerdes);
    diskOrderScan(indexAccessor, fieldSerdes);
    // Build low key.
    ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
    ArrayTupleReference lowKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
    // Build high key.
    ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
    ArrayTupleReference highKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
    rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 12 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class OrderedIndexExamplesTest method varLenKeyValueExample.

/**
     * Variable-Length Example. Create a BTree with one variable-length key
     * field and one variable-length value field. Fill BTree with random values
     * using insertions (not bulk load) Perform ordered scans and range search.
     */
@Test
public void varLenKeyValueExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Variable-Length Key,Value Example");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
    typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // Max string length to be generated.
    int maxLength = 10;
    int numInserts = 10000;
    for (int i = 0; i < 10000; i++) {
        String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 1000 == 0) {
                LOGGER.info("Inserting[" + i + "] " + f0 + " " + f1);
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
    }
    orderedScan(indexAccessor, fieldSerdes);
    diskOrderScan(indexAccessor, fieldSerdes);
    // Build low key.
    ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference lowKey = new ArrayTupleReference();
    TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
    // Build high key.
    ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference highKey = new ArrayTupleReference();
    TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
    rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 13 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class DatasetUtil method createDummyKeyProviderOp.

/**
     * Creates a dummy key provider operator for the primary index scan.
     *
     * @param spec,
     *            the job specification.
     * @param dataset,
     *            the dataset to scan.
     * @param metadataProvider,
     *            the metadata provider.
     * @return a dummy key provider operator.
     * @throws AlgebricksException
     */
public static IOperatorDescriptor createDummyKeyProviderOp(JobSpecification spec, Dataset dataset, MetadataProvider metadataProvider) throws AlgebricksException {
    Pair<IFileSplitProvider, AlgebricksPartitionConstraint> primarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset);
    AlgebricksPartitionConstraint primaryPartitionConstraint = primarySplitsAndConstraint.second;
    // Build dummy tuple containing one field with a dummy value inside.
    ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
    DataOutput dos = tb.getDataOutput();
    tb.reset();
    try {
        // Serialize dummy value into a field.
        IntegerSerializerDeserializer.INSTANCE.serialize(0, dos);
    } catch (HyracksDataException e) {
        throw new AsterixException(e);
    }
    // Add dummy field.
    tb.addFieldEndOffset();
    ISerializerDeserializer[] keyRecDescSers = { IntegerSerializerDeserializer.INSTANCE };
    RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
    ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, keyProviderOp, primaryPartitionConstraint);
    return keyProviderOp;
}

Example 14 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class BinaryConcatDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
            return new AbstractBinaryScalarEvaluator(ctx, args) {

                private final ListAccessor listAccessor = new ListAccessor();

                private final byte[] metaBuffer = new byte[5];

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<ANull> nullSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ANULL);

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<AMissing> missingSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AMISSING);

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    evaluators[0].evaluate(tuple, pointables[0]);
                    byte[] data = pointables[0].getByteArray();
                    int offset = pointables[0].getStartOffset();
                    byte typeTag = data[offset];
                    if (typeTag != ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG && typeTag != ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG) {
                        throw new TypeMismatchException(getIdentifier(), 0, typeTag, ATypeTag.SERIALIZED_UNORDEREDLIST_TYPE_TAG, ATypeTag.SERIALIZED_ORDEREDLIST_TYPE_TAG);
                    }
                    try {
                        listAccessor.reset(data, offset);
                        int concatLength = 0;
                        for (int i = 0; i < listAccessor.size(); i++) {
                            int itemOffset = listAccessor.getItemOffset(i);
                            ATypeTag itemType = listAccessor.getItemType(itemOffset);
                            if (itemType != ATypeTag.BINARY) {
                                if (serializeUnknownIfAnyUnknown(itemType)) {
                                    result.set(resultStorage);
                                    return;
                                }
                                throw new UnsupportedItemTypeException(getIdentifier(), itemType.serialize());
                            }
                            concatLength += ByteArrayPointable.getContentLength(data, itemOffset);
                        }
                        dataOutput.writeByte(ATypeTag.SERIALIZED_BINARY_TYPE_TAG);
                        int metaLen = VarLenIntEncoderDecoder.encode(concatLength, metaBuffer, 0);
                        dataOutput.write(metaBuffer, 0, metaLen);
                        for (int i = 0; i < listAccessor.size(); i++) {
                            int itemOffset = listAccessor.getItemOffset(i);
                            int length = ByteArrayPointable.getContentLength(data, itemOffset);
                            dataOutput.write(data, itemOffset + ByteArrayPointable.getNumberBytesToStoreMeta(length), length);
                        }
                    } catch (IOException e) {
                        throw new HyracksDataException(e);
                    } catch (AsterixException e) {
                        throw new HyracksDataException(e);
                    }
                    result.set(resultStorage);
                }

                private boolean serializeUnknownIfAnyUnknown(ATypeTag... tags) throws HyracksDataException {
                    for (ATypeTag typeTag : tags) {
                        if (typeTag == ATypeTag.NULL) {
                            nullSerde.serialize(ANull.NULL, dataOutput);
                            return true;
                        }
                        if (typeTag == ATypeTag.MISSING) {
                            missingSerde.serialize(AMissing.MISSING, dataOutput);
                            return true;
                        }
                    }
                    return false;
                }
            };
        }
    };
}

Example 15 with ISerializerDeserializer

use of org.apache.hyracks.api.dataflow.value.ISerializerDeserializer in project asterixdb by apache.

the class ParseBinaryDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
            return new AbstractBinaryScalarEvaluator(ctx, args) {

                @SuppressWarnings("unchecked")
                private ISerializerDeserializer<ABinary> binarySerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ABINARY);

                private AMutableBinary aBinary = new AMutableBinary(new byte[0], 0, 0);

                private final UTF8StringPointable stringPointable = new UTF8StringPointable();

                private final UTF8StringPointable formatPointable = new UTF8StringPointable();

                private final HexParser hexParser = new HexParser();

                private final Base64Parser base64Parser = new Base64Parser();

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    resultStorage.reset();
                    evaluators[0].evaluate(tuple, pointables[0]);
                    evaluators[1].evaluate(tuple, pointables[1]);
                    ATypeTag binaryTag = ATypeTag.VALUE_TYPE_MAPPING[pointables[0].getByteArray()[pointables[0].getStartOffset()]];
                    ATypeTag formatTag = ATypeTag.VALUE_TYPE_MAPPING[pointables[1].getByteArray()[pointables[1].getStartOffset()]];
                    checkTypeMachingThrowsIfNot(getIdentifier().getName(), EXPECTED_INPUT_TAGS, binaryTag, formatTag);
                    stringPointable.set(pointables[0].getByteArray(), pointables[0].getStartOffset() + 1, pointables[0].getLength());
                    formatPointable.set(pointables[1].getByteArray(), pointables[1].getStartOffset() + 1, pointables[1].getLength());
                    if (HEX_FORMAT.ignoreCaseCompareTo(formatPointable) == 0) {
                        hexParser.generateByteArrayFromHexString(stringPointable.getByteArray(), stringPointable.getCharStartOffset(), stringPointable.getUTF8Length());
                        aBinary.setValue(hexParser.getByteArray(), 0, hexParser.getLength());
                    } else if (BASE64_FORMAT.ignoreCaseCompareTo(formatPointable) == 0) {
                        base64Parser.generatePureByteArrayFromBase64String(stringPointable.getByteArray(), stringPointable.getCharStartOffset(), stringPointable.getUTF8Length());
                        aBinary.setValue(base64Parser.getByteArray(), 0, base64Parser.getLength());
                    } else {
                        throw new UnsupportedItemTypeException(getIdentifier(), formatTag.serialize());
                    }
                    binarySerde.serialize(aBinary, dataOutput);
                    result.set(resultStorage);
                }
            };
        }
    };
}