Examples with AString org.apache.asterix.om.base.AString used on opensource projects

Example 41 with AString

use of org.apache.asterix.om.base.AString in project asterixdb by apache.

the class ARecordSerializerDeserializer method serializeSimpleSchemalessRecord.

@SuppressWarnings("unchecked")
public static void serializeSimpleSchemalessRecord(List<Pair<String, String>> record, DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
    ISerializerDeserializer<AString> stringSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
    RecordBuilder confRecordBuilder = new RecordBuilder();
    confRecordBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE);
    ArrayBackedValueStorage fieldNameBytes = new ArrayBackedValueStorage();
    ArrayBackedValueStorage fieldValueBytes = new ArrayBackedValueStorage();
    for (int i = 0; i < record.size(); i++) {
        fieldValueBytes.reset();
        fieldNameBytes.reset();
        stringSerde.serialize(new AString(record.get(i).first), fieldNameBytes.getDataOutput());
        stringSerde.serialize(new AString(record.get(i).second), fieldValueBytes.getDataOutput());
        confRecordBuilder.addField(fieldNameBytes, fieldValueBytes);
    }
    confRecordBuilder.write(dataOutput, writeTypeTag);
}

Example 42 with AString

use of org.apache.asterix.om.base.AString in project asterixdb by apache.

the class ARecordSerializerDeserializer method serializeSchemalessRecord.

@SuppressWarnings({ "unchecked", "rawtypes" })
public static void serializeSchemalessRecord(ARecord record, DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException {
    ISerializerDeserializer<AString> stringSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
    RecordBuilder confRecordBuilder = new RecordBuilder();
    confRecordBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE);
    ArrayBackedValueStorage fieldNameBytes = new ArrayBackedValueStorage();
    ArrayBackedValueStorage fieldValueBytes = new ArrayBackedValueStorage();
    for (int i = 0; i < record.getType().getFieldNames().length; i++) {
        String fieldName = record.getType().getFieldNames()[i];
        fieldValueBytes.reset();
        fieldNameBytes.reset();
        stringSerde.serialize(new AString(fieldName), fieldNameBytes.getDataOutput());
        ISerializerDeserializer valueSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(record.getType().getFieldTypes()[i]);
        valueSerde.serialize(record.getValueByPos(i), fieldValueBytes.getDataOutput());
        confRecordBuilder.addField(fieldNameBytes, fieldValueBytes);
    }
    confRecordBuilder.write(dataOutput, writeTypeTag);
}

Example 43 with AString

use of org.apache.asterix.om.base.AString in project asterixdb by apache.

the class RecordPairsDescriptor method createEvaluatorFactory.

@Override
public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) {
    return new IScalarEvaluatorFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public IScalarEvaluator createScalarEvaluator(IHyracksTaskContext ctx) throws HyracksDataException {
            // For writing each individual output record.
            final ArrayBackedValueStorage itemStorage = new ArrayBackedValueStorage();
            final DataOutput itemOutput = itemStorage.getDataOutput();
            final RecordBuilder recBuilder = new RecordBuilder();
            recBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE);
            // For writing the resulting list of records.
            final OrderedListBuilder listBuilder = new OrderedListBuilder();
            final ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();
            final DataOutput resultOut = resultStorage.getDataOutput();
            // Sets up the constant field names, "name" for the key field, "value" for the value field.
            final ArrayBackedValueStorage nameStorage = new ArrayBackedValueStorage();
            final ArrayBackedValueStorage valueStorage = new ArrayBackedValueStorage();
            AObjectSerializerDeserializer serde = AObjectSerializerDeserializer.INSTANCE;
            try {
                serde.serialize(new AString("name"), nameStorage.getDataOutput());
                serde.serialize(new AString("value"), valueStorage.getDataOutput());
            } catch (IOException e) {
                throw new HyracksDataException(e);
            }
            return new IScalarEvaluator() {

                private final IScalarEvaluator argEvaluator = args[0].createScalarEvaluator(ctx);

                private final IPointable argPtr = new VoidPointable();

                private final ARecordVisitablePointable recordVisitablePointable = new ARecordVisitablePointable(recType);

                @Override
                public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
                    // Resets the result storage.
                    resultStorage.reset();
                    // Gets the input record.
                    argEvaluator.evaluate(tuple, argPtr);
                    byte inputTypeTag = argPtr.getByteArray()[argPtr.getStartOffset()];
                    if (inputTypeTag != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
                        throw new TypeMismatchException(getIdentifier(), 0, inputTypeTag, ATypeTag.SERIALIZED_RECORD_TYPE_TAG);
                    }
                    recordVisitablePointable.set(argPtr);
                    listBuilder.reset(AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE);
                    List<IVisitablePointable> fieldNames = recordVisitablePointable.getFieldNames();
                    List<IVisitablePointable> fieldValues = recordVisitablePointable.getFieldValues();
                    // Adds each field of the input record as a key-value pair into the result.
                    int numFields = recordVisitablePointable.getFieldNames().size();
                    for (int fieldIndex = 0; fieldIndex < numFields; ++fieldIndex) {
                        itemStorage.reset();
                        recBuilder.init();
                        recBuilder.addField(nameStorage, fieldNames.get(fieldIndex));
                        recBuilder.addField(valueStorage, fieldValues.get(fieldIndex));
                        recBuilder.write(itemOutput, true);
                        listBuilder.addItem(itemStorage);
                    }
                    // Writes the result and sets the result pointable.
                    listBuilder.write(resultOut, true);
                    result.set(resultStorage);
                }
            };
        }
    };
}

Example 44 with AString

use of org.apache.asterix.om.base.AString in project asterixdb by apache.

the class FieldAccessNestedEvalFactory method createScalarEvaluator.

@Override
public IScalarEvaluator createScalarEvaluator(final IHyracksTaskContext ctx) throws HyracksDataException {
    return new IScalarEvaluator() {

        private ArrayBackedValueStorage resultStorage = new ArrayBackedValueStorage();

        private final DataOutput out = resultStorage.getDataOutput();

        private final ByteArrayAccessibleOutputStream subRecordTmpStream = new ByteArrayAccessibleOutputStream();

        private final IPointable inputArg0 = new VoidPointable();

        private final IScalarEvaluator eval0 = recordEvalFactory.createScalarEvaluator(ctx);

        private final IPointable[] fieldPointables = new VoidPointable[fieldPath.size()];

        private final RuntimeRecordTypeInfo[] recTypeInfos = new RuntimeRecordTypeInfo[fieldPath.size()];

        @SuppressWarnings("unchecked")
        private final ISerializerDeserializer<ANull> nullSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ANULL);

        @SuppressWarnings("unchecked")
        private final ISerializerDeserializer<AMissing> missingSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AMISSING);

        {
            generateFieldsPointables();
            for (int index = 0; index < fieldPath.size(); ++index) {
                recTypeInfos[index] = new RuntimeRecordTypeInfo();
            }
        }

        @SuppressWarnings("unchecked")
        private void generateFieldsPointables() throws HyracksDataException {
            for (int i = 0; i < fieldPath.size(); i++) {
                ArrayBackedValueStorage storage = new ArrayBackedValueStorage();
                DataOutput out = storage.getDataOutput();
                AString as = new AString(fieldPath.get(i));
                SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(as.getType()).serialize(as, out);
                fieldPointables[i] = new VoidPointable();
                fieldPointables[i].set(storage);
            }
        }

        @Override
        public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException {
            try {
                resultStorage.reset();
                eval0.evaluate(tuple, inputArg0);
                byte[] serRecord = inputArg0.getByteArray();
                int offset = inputArg0.getStartOffset();
                int start = offset;
                int len = inputArg0.getLength();
                if (serRecord[start] != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
                    throw new TypeMismatchException(BuiltinFunctions.FIELD_ACCESS_NESTED, 0, serRecord[start], ATypeTag.SERIALIZED_RECORD_TYPE_TAG);
                }
                int subFieldIndex = -1;
                int subFieldOffset = -1;
                int subFieldLength = -1;
                int nullBitmapSize = -1;
                IAType subType = recordType;
                recTypeInfos[0].reset(recordType);
                ATypeTag subTypeTag = ATypeTag.MISSING;
                boolean openField = false;
                int pathIndex = 0;
                // Moving through closed fields first.
                for (; pathIndex < fieldPointables.length; pathIndex++) {
                    if (subType.getTypeTag().equals(ATypeTag.UNION)) {
                        //enforced SubType
                        subType = ((AUnionType) subType).getActualType();
                        byte serializedTypeTag = subType.getTypeTag().serialize();
                        if (serializedTypeTag != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
                            throw new UnsupportedTypeException(BuiltinFunctions.FIELD_ACCESS_NESTED.getName(), serializedTypeTag);
                        }
                        if (subType.getTypeTag() == ATypeTag.OBJECT) {
                            recTypeInfos[pathIndex].reset((ARecordType) subType);
                        }
                    }
                    subFieldIndex = recTypeInfos[pathIndex].getFieldIndex(fieldPointables[pathIndex].getByteArray(), fieldPointables[pathIndex].getStartOffset() + 1, fieldPointables[pathIndex].getLength() - 1);
                    if (subFieldIndex == -1) {
                        break;
                    }
                    nullBitmapSize = RecordUtil.computeNullBitmapSize((ARecordType) subType);
                    subFieldOffset = ARecordSerializerDeserializer.getFieldOffsetById(serRecord, start, subFieldIndex, nullBitmapSize, ((ARecordType) subType).isOpen());
                    if (subFieldOffset == 0) {
                        // the field is null, we checked the null bit map
                        // any path after null will return null.
                        nullSerde.serialize(ANull.NULL, out);
                        result.set(resultStorage);
                        return;
                    }
                    if (subFieldOffset < 0) {
                        // the field is missing, we checked the missing bit map
                        // any path after missing will return null.
                        missingSerde.serialize(AMissing.MISSING, out);
                        result.set(resultStorage);
                        return;
                    }
                    subType = ((ARecordType) subType).getFieldTypes()[subFieldIndex];
                    if (subType.getTypeTag() == ATypeTag.OBJECT && pathIndex + 1 < fieldPointables.length) {
                        // Move to the next Depth
                        recTypeInfos[pathIndex + 1].reset((ARecordType) subType);
                    }
                    if (subType.getTypeTag().equals(ATypeTag.UNION)) {
                        subTypeTag = ((AUnionType) subType).getActualType().getTypeTag();
                        subFieldLength = NonTaggedFormatUtil.getFieldValueLength(serRecord, subFieldOffset, subTypeTag, false);
                    } else {
                        subTypeTag = subType.getTypeTag();
                        subFieldLength = NonTaggedFormatUtil.getFieldValueLength(serRecord, subFieldOffset, subTypeTag, false);
                    }
                    if (pathIndex < fieldPointables.length - 1) {
                        //setup next iteration
                        subRecordTmpStream.reset();
                        subRecordTmpStream.write(subTypeTag.serialize());
                        subRecordTmpStream.write(serRecord, subFieldOffset, subFieldLength);
                        serRecord = subRecordTmpStream.getByteArray();
                        start = 0;
                    }
                    // type check
                    if (pathIndex < fieldPointables.length - 1 && serRecord[start] != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
                        throw new UnsupportedTypeException(BuiltinFunctions.FIELD_ACCESS_NESTED, serRecord[start]);
                    }
                }
                // Moving through open fields after we hit the first open field.
                for (; pathIndex < fieldPointables.length; pathIndex++) {
                    openField = true;
                    subFieldOffset = ARecordSerializerDeserializer.getFieldOffsetByName(serRecord, start, len, fieldPointables[pathIndex].getByteArray(), fieldPointables[pathIndex].getStartOffset());
                    if (subFieldOffset < 0) {
                        out.writeByte(ATypeTag.SERIALIZED_MISSING_TYPE_TAG);
                        result.set(resultStorage);
                        return;
                    }
                    subTypeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(serRecord[subFieldOffset]);
                    subFieldLength = NonTaggedFormatUtil.getFieldValueLength(serRecord, subFieldOffset, subTypeTag, true) + 1;
                    if (pathIndex >= fieldPointables.length - 1) {
                        continue;
                    }
                    //setup next iteration
                    start = subFieldOffset;
                    len = subFieldLength;
                    // type check
                    if (serRecord[start] == ATypeTag.SERIALIZED_MISSING_TYPE_TAG) {
                        missingSerde.serialize(AMissing.MISSING, out);
                        result.set(resultStorage);
                        return;
                    }
                    if (serRecord[start] != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
                        throw new UnsupportedTypeException(BuiltinFunctions.FIELD_ACCESS_NESTED.getName(), serRecord[start]);
                    }
                }
                // emit the final result.
                if (openField) {
                    result.set(serRecord, subFieldOffset, subFieldLength);
                } else {
                    out.writeByte(subTypeTag.serialize());
                    out.write(serRecord, subFieldOffset, subFieldLength);
                    result.set(resultStorage);
                }
            } catch (IOException | AsterixException e) {
                throw new HyracksDataException(e);
            }
        }
    };
}

Example 45 with AString

use of org.apache.asterix.om.base.AString in project asterixdb by apache.

the class IndexTupleTranslator method getTupleFromMetadataEntity.

@Override
public ITupleReference getTupleFromMetadataEntity(Index instance) throws HyracksDataException {
    // write the key in the first 3 fields of the tuple
    tupleBuilder.reset();
    aString.setValue(instance.getDataverseName());
    stringSerde.serialize(aString, tupleBuilder.getDataOutput());
    tupleBuilder.addFieldEndOffset();
    aString.setValue(instance.getDatasetName());
    stringSerde.serialize(aString, tupleBuilder.getDataOutput());
    tupleBuilder.addFieldEndOffset();
    aString.setValue(instance.getIndexName());
    stringSerde.serialize(aString, tupleBuilder.getDataOutput());
    tupleBuilder.addFieldEndOffset();
    // write the payload in the fourth field of the tuple
    recordBuilder.reset(MetadataRecordTypes.INDEX_RECORDTYPE);
    // write field 0
    fieldValue.reset();
    aString.setValue(instance.getDataverseName());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_DATAVERSENAME_FIELD_INDEX, fieldValue);
    // write field 1
    fieldValue.reset();
    aString.setValue(instance.getDatasetName());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_DATASETNAME_FIELD_INDEX, fieldValue);
    // write field 2
    fieldValue.reset();
    aString.setValue(instance.getIndexName());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_INDEXNAME_FIELD_INDEX, fieldValue);
    // write field 3
    fieldValue.reset();
    aString.setValue(instance.getIndexType().toString());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_INDEXSTRUCTURE_FIELD_INDEX, fieldValue);
    // write field 4
    primaryKeyListBuilder.reset((AOrderedListType) MetadataRecordTypes.INDEX_RECORDTYPE.getFieldTypes()[MetadataRecordTypes.INDEX_ARECORD_SEARCHKEY_FIELD_INDEX]);
    List<List<String>> searchKey = instance.getKeyFieldNames();
    for (List<String> field : searchKey) {
        listBuilder.reset(stringList);
        for (String subField : field) {
            itemValue.reset();
            aString.setValue(subField);
            stringSerde.serialize(aString, itemValue.getDataOutput());
            listBuilder.addItem(itemValue);
        }
        itemValue.reset();
        listBuilder.write(itemValue.getDataOutput(), true);
        primaryKeyListBuilder.addItem(itemValue);
    }
    fieldValue.reset();
    primaryKeyListBuilder.write(fieldValue.getDataOutput(), true);
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_SEARCHKEY_FIELD_INDEX, fieldValue);
    // write field 5
    fieldValue.reset();
    if (instance.isPrimaryIndex()) {
        booleanSerde.serialize(ABoolean.TRUE, fieldValue.getDataOutput());
    } else {
        booleanSerde.serialize(ABoolean.FALSE, fieldValue.getDataOutput());
    }
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_ISPRIMARY_FIELD_INDEX, fieldValue);
    // write field 6
    fieldValue.reset();
    aString.setValue(Calendar.getInstance().getTime().toString());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_TIMESTAMP_FIELD_INDEX, fieldValue);
    // write field 7
    fieldValue.reset();
    intSerde.serialize(new AInt32(instance.getPendingOp()), fieldValue.getDataOutput());
    recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_PENDINGOP_FIELD_INDEX, fieldValue);
    // write optional field 8
    if (instance.getGramLength() > 0) {
        fieldValue.reset();
        nameValue.reset();
        aString.setValue(GRAM_LENGTH_FIELD_NAME);
        stringSerde.serialize(aString, nameValue.getDataOutput());
        intSerde.serialize(new AInt32(instance.getGramLength()), fieldValue.getDataOutput());
        recordBuilder.addField(nameValue, fieldValue);
    }
    if (instance.isEnforcingKeyFileds()) {
        // write optional field 9
        OrderedListBuilder typeListBuilder = new OrderedListBuilder();
        typeListBuilder.reset(new AOrderedListType(BuiltinType.ANY, null));
        nameValue.reset();
        aString.setValue(INDEX_SEARCHKEY_TYPE_FIELD_NAME);
        stringSerde.serialize(aString, nameValue.getDataOutput());
        List<IAType> searchKeyType = instance.getKeyFieldTypes();
        for (IAType type : searchKeyType) {
            itemValue.reset();
            aString.setValue(type.getTypeName());
            stringSerde.serialize(aString, itemValue.getDataOutput());
            typeListBuilder.addItem(itemValue);
        }
        fieldValue.reset();
        typeListBuilder.write(fieldValue.getDataOutput(), true);
        recordBuilder.addField(nameValue, fieldValue);
        // write optional field 10
        fieldValue.reset();
        nameValue.reset();
        aString.setValue(INDEX_ISENFORCED_FIELD_NAME);
        stringSerde.serialize(aString, nameValue.getDataOutput());
        booleanSerde.serialize(ABoolean.TRUE, fieldValue.getDataOutput());
        recordBuilder.addField(nameValue, fieldValue);
    }
    List<Integer> keySourceIndicator = instance.getKeyFieldSourceIndicators();
    boolean needSerialization = false;
    if (keySourceIndicator != null) {
        for (int source : keySourceIndicator) {
            if (source != 0) {
                needSerialization = true;
                break;
            }
        }
    }
    if (needSerialization) {
        listBuilder.reset(int8List);
        nameValue.reset();
        aString.setValue(INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME);
        stringSerde.serialize(aString, nameValue.getDataOutput());
        for (int source : keySourceIndicator) {
            itemValue.reset();
            aInt8.setValue((byte) source);
            int8Serde.serialize(aInt8, itemValue.getDataOutput());
            listBuilder.addItem(itemValue);
        }
        fieldValue.reset();
        listBuilder.write(fieldValue.getDataOutput(), true);
        recordBuilder.addField(nameValue, fieldValue);
    }
    // write record
    recordBuilder.write(tupleBuilder.getDataOutput(), true);
    tupleBuilder.addFieldEndOffset();
    tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
    return tuple;
}