Examples with AUnionType org.apache.asterix.om.types.AUnionType used on opensource projects

Example 6 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class HiveRecordParser method getHiveTypeString.

private String getHiveTypeString(IAType[] types, int i) throws HyracksDataException {
    final IAType type = types[i];
    ATypeTag tag = type.getTypeTag();
    if (tag == ATypeTag.UNION) {
        if (NonTaggedFormatUtil.isOptional(type)) {
            throw new RuntimeDataException(ErrorCode.PARSER_HIVE_NOT_SUPPORT_NON_OP_UNION);
        }
        tag = ((AUnionType) type).getActualType().getTypeTag();
    }
    if (tag == null) {
        throw new RuntimeDataException(ErrorCode.PARSER_HIVE_MISSING_FIELD_TYPE_INFO, i);
    }
    switch(tag) {
        case BOOLEAN:
            return Constants.BOOLEAN_TYPE_NAME;
        case DATE:
            return Constants.DATE_TYPE_NAME;
        case DATETIME:
            return Constants.DATETIME_TYPE_NAME;
        case DOUBLE:
            return Constants.DOUBLE_TYPE_NAME;
        case FLOAT:
            return Constants.FLOAT_TYPE_NAME;
        case SMALLINT:
            return Constants.SMALLINT_TYPE_NAME;
        case INTEGER:
            return Constants.INT_TYPE_NAME;
        case BIGINT:
            return Constants.BIGINT_TYPE_NAME;
        case TINYINT:
            return Constants.TINYINT_TYPE_NAME;
        case ARRAY:
            return Constants.LIST_TYPE_NAME;
        case STRING:
            return Constants.STRING_TYPE_NAME;
        case TIME:
            return Constants.DATETIME_TYPE_NAME;
        case MULTISET:
            return Constants.LIST_TYPE_NAME;
        default:
            throw new RuntimeDataException(ErrorCode.PARSER_HIVE_FIELD_TYPE, tag);
    }
}

Example 7 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class JObjectUtil method getJType.

public static IJObject getJType(ATypeTag typeTag, IAType type, ByteArrayAccessibleDataInputStream dis, IObjectPool<IJObject, IAType> objectPool) throws IOException {
    IJObject jObject;
    switch(typeTag) {
        case INTEGER:
            {
                int v = dis.readInt();
                jObject = objectPool.allocate(BuiltinType.AINT32);
                ((JInt) jObject).setValue(v);
                break;
            }
        case FLOAT:
            {
                float v = dis.readFloat();
                jObject = objectPool.allocate(BuiltinType.AFLOAT);
                ((JFloat) jObject).setValue(v);
                break;
            }
        case DOUBLE:
            {
                double value = dis.readDouble();
                jObject = objectPool.allocate(BuiltinType.ADOUBLE);
                ((JDouble) jObject).setValue(value);
                break;
            }
        case STRING:
            {
                String v = dis.readUTF();
                jObject = objectPool.allocate(BuiltinType.ASTRING);
                ((JString) jObject).setValue(v);
                break;
            }
        case BOOLEAN:
            jObject = objectPool.allocate(BuiltinType.ABOOLEAN);
            ((JBoolean) jObject).setValue(dis.readBoolean());
            break;
        case DATE:
            {
                int d = dis.readInt();
                jObject = objectPool.allocate(BuiltinType.ADATE);
                ((JDate) jObject).setValue(d);
                break;
            }
        case DATETIME:
            {
                jObject = objectPool.allocate(BuiltinType.ADATETIME);
                long value = dis.readLong();
                ((JDateTime) jObject).setValue(value);
                break;
            }
        case DURATION:
            {
                jObject = objectPool.allocate(BuiltinType.ADURATION);
                int months = dis.readInt();
                long msecs = dis.readLong();
                ((JDuration) jObject).setValue(months, msecs);
                break;
            }
        case TIME:
            {
                jObject = objectPool.allocate(BuiltinType.ATIME);
                int time = dis.readInt();
                ((JTime) jObject).setValue(time);
                break;
            }
        case INTERVAL:
            {
                jObject = objectPool.allocate(BuiltinType.AINTERVAL);
                long start = dis.readLong();
                long end = dis.readLong();
                byte intervalType = dis.readByte();
                ((JInterval) jObject).setValue(start, end, intervalType);
                break;
            }
        case CIRCLE:
            {
                jObject = objectPool.allocate(BuiltinType.ACIRCLE);
                double x = dis.readDouble();
                double y = dis.readDouble();
                double radius = dis.readDouble();
                JPoint jpoint = (JPoint) objectPool.allocate(BuiltinType.APOINT);
                jpoint.setValue(x, y);
                ((JCircle) jObject).setValue(jpoint, radius);
                break;
            }
        case POINT:
            {
                jObject = objectPool.allocate(BuiltinType.APOINT);
                double x = dis.readDouble();
                double y = dis.readDouble();
                ((JPoint) jObject).setValue(x, y);
                break;
            }
        case POINT3D:
            {
                jObject = objectPool.allocate(BuiltinType.APOINT3D);
                double x = dis.readDouble();
                double y = dis.readDouble();
                double z = dis.readDouble();
                ((JPoint3D) jObject).setValue(x, y, z);
                break;
            }
        case LINE:
            {
                jObject = objectPool.allocate(BuiltinType.ALINE);
                double x1 = dis.readDouble();
                double y1 = dis.readDouble();
                double x2 = dis.readDouble();
                double y2 = dis.readDouble();
                JPoint jpoint1 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
                jpoint1.setValue(x1, y1);
                JPoint jpoint2 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
                jpoint2.setValue(x2, y2);
                ((JLine) jObject).setValue(jpoint1, jpoint2);
                break;
            }
        case POLYGON:
            {
                jObject = objectPool.allocate(BuiltinType.APOLYGON);
                short numberOfPoints = dis.readShort();
                List<JPoint> points = new ArrayList<JPoint>();
                for (int i = 0; i < numberOfPoints; i++) {
                    JPoint p1 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
                    p1.setValue(dis.readDouble(), dis.readDouble());
                    points.add(p1);
                }
                ((JPolygon) jObject).setValue(points.toArray(new APoint[] {}));
                break;
            }
        case RECTANGLE:
            {
                jObject = objectPool.allocate(BuiltinType.ARECTANGLE);
                double x1 = dis.readDouble();
                double y1 = dis.readDouble();
                double x2 = dis.readDouble();
                double y2 = dis.readDouble();
                JPoint jpoint1 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
                jpoint1.setValue(x1, y1);
                JPoint jpoint2 = (JPoint) objectPool.allocate(BuiltinType.APOINT);
                jpoint2.setValue(x2, y2);
                ((JRectangle) jObject).setValue(jpoint1, jpoint2);
                break;
            }
        case MULTISET:
            {
                AUnorderedListType listType = (AUnorderedListType) type;
                IAType elementType = listType.getItemType();
                jObject = objectPool.allocate(listType);
                boolean fixedSize = false;
                ATypeTag tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(dis.readByte());
                switch(tag) {
                    case STRING:
                    case OBJECT:
                    case ARRAY:
                    case MULTISET:
                    case ANY:
                        fixedSize = false;
                        break;
                    default:
                        fixedSize = true;
                        break;
                }
                // list size
                dis.readInt();
                int numberOfitems;
                numberOfitems = dis.readInt();
                if (numberOfitems <= 0) {
                    break;
                }
                if (!fixedSize) {
                    for (int i = 0; i < numberOfitems; i++) {
                        dis.readInt();
                    }
                }
                for (int i = 0; i < numberOfitems; i++) {
                    IJObject v = getJType(elementType.getTypeTag(), elementType, dis, objectPool);
                    ((JUnorderedList) jObject).add(v);
                }
                break;
            }
        case ARRAY:
            {
                AOrderedListType listType = (AOrderedListType) type;
                IAType elementType = listType.getItemType();
                jObject = objectPool.allocate(listType);
                boolean fixedSize = false;
                ATypeTag tag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(dis.readByte());
                switch(tag) {
                    case STRING:
                    case OBJECT:
                    case ARRAY:
                    case MULTISET:
                    case ANY:
                        fixedSize = false;
                        break;
                    default:
                        fixedSize = true;
                        break;
                }
                // list size
                dis.readInt();
                int numberOfitems;
                numberOfitems = dis.readInt();
                if (numberOfitems <= 0) {
                    break;
                }
                if (!fixedSize) {
                    for (int i = 0; i < numberOfitems; i++) {
                        dis.readInt();
                    }
                }
                for (int i = 0; i < numberOfitems; i++) {
                    IJObject v = getJType(elementType.getTypeTag(), elementType, dis, objectPool);
                    ((JOrderedList) jObject).add(v);
                }
                break;
            }
        case OBJECT:
            ARecordType recordType = (ARecordType) type;
            int numberOfSchemaFields = recordType.getFieldTypes().length;
            byte[] recordBits = dis.getInputStream().getArray();
            boolean isExpanded = false;
            dis.getInputStream();
            int[] fieldOffsets = new int[numberOfSchemaFields];
            IJObject[] closedFields = new IJObject[numberOfSchemaFields];
            // reading length is not required.
            dis.skip(4);
            if (recordType.isOpen()) {
                isExpanded = dis.readBoolean();
                if (isExpanded) {
                    dis.readInt();
                } else {
                }
            } else {
            }
            if (numberOfSchemaFields > 0) {
                dis.readInt();
                int nullBitMapOffset = 0;
                boolean hasOptionalFields = NonTaggedFormatUtil.hasOptionalField(recordType);
                if (hasOptionalFields) {
                    nullBitMapOffset = dis.getInputStream().getPosition();
                    dis.getInputStream();
                } else {
                    dis.getInputStream();
                }
                for (int i = 0; i < numberOfSchemaFields; i++) {
                    fieldOffsets[i] = dis.readInt();
                }
                for (int fieldNumber = 0; fieldNumber < numberOfSchemaFields; fieldNumber++) {
                    if (hasOptionalFields) {
                        byte b1 = recordBits[nullBitMapOffset + fieldNumber / 8];
                        int p = 1 << (7 - (fieldNumber % 8));
                        if ((b1 & p) == 0) {
                            continue;
                        }
                    }
                    IAType[] fieldTypes = recordType.getFieldTypes();
                    ATypeTag fieldValueTypeTag = null;
                    IAType fieldType = fieldTypes[fieldNumber];
                    if (fieldTypes[fieldNumber].getTypeTag() == ATypeTag.UNION) {
                        if (((AUnionType) fieldTypes[fieldNumber]).isUnknownableType()) {
                            fieldType = ((AUnionType) fieldTypes[fieldNumber]).getActualType();
                            fieldValueTypeTag = fieldType.getTypeTag();
                        }
                    } else {
                        fieldValueTypeTag = fieldTypes[fieldNumber].getTypeTag();
                    }
                    closedFields[fieldNumber] = getJType(fieldValueTypeTag, fieldType, dis, objectPool);
                }
            }
            if (isExpanded) {
                int numberOfOpenFields = dis.readInt();
                String[] fieldNames = new String[numberOfOpenFields];
                IAType[] fieldTypes = new IAType[numberOfOpenFields];
                IJObject[] openFields = new IJObject[numberOfOpenFields];
                for (int i = 0; i < numberOfOpenFields; i++) {
                    dis.readInt();
                    dis.readInt();
                }
                for (int i = 0; i < numberOfOpenFields; i++) {
                    fieldNames[i] = AStringSerializerDeserializer.INSTANCE.deserialize(dis).getStringValue();
                    ATypeTag openFieldTypeTag = SerializerDeserializerUtil.deserializeTag(dis);
                    openFields[i] = getJType(openFieldTypeTag, null, dis, objectPool);
                    fieldTypes[i] = openFields[i].getIAObject().getType();
                }
                ARecordType openPartRecType = new ARecordType(null, fieldNames, fieldTypes, true);
                if (numberOfSchemaFields > 0) {
                    ARecordType mergedRecordType = mergeRecordTypes(recordType, openPartRecType);
                    IJObject[] mergedFields = mergeFields(closedFields, openFields);
                    jObject = objectPool.allocate(recordType);
                    return new JRecord(mergedRecordType, mergedFields);
                } else {
                    return new JRecord(recordType, openFields);
                }
            } else {
                return new JRecord(recordType, closedFields);
            }
        default:
            throw new RuntimeDataException(ErrorCode.LIBRARY_JOBJECT_UTIL_ILLEGAL_ARGU_TYPE, typeTag);
    }
    return jObject;
}

Example 8 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class ADMDataParser method checkOptionalConstraints.

private int checkOptionalConstraints(ARecordType recType, BitSet nulls) {
    for (int i = 0; i < recType.getFieldTypes().length; i++) {
        if (nulls.get(i) == false) {
            IAType type = recType.getFieldTypes()[i];
            if ((type.getTypeTag() != ATypeTag.NULL) && (type.getTypeTag() != ATypeTag.UNION)) {
                return i;
            }
            if (type.getTypeTag() != ATypeTag.UNION) {
                continue;
            }
            // union
            AUnionType unionType = (AUnionType) type;
            if (!unionType.isUnknownableType()) {
                return i;
            }
        }
    }
    return -1;
}

Example 9 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class ARecordVisitablePointable method set.

@Override
public void set(byte[] b, int start, int len) {
    // clear the previous states
    reset();
    super.set(b, start, len);
    boolean isExpanded = false;
    int openPartOffset = 0;
    int recordOffset = start;
    int offsetArrayOffset;
    //advance to either isExpanded or numberOfSchemaFields
    int s = start + 5;
    //inputRecType will never be null.
    if (inputRecType.isOpen()) {
        isExpanded = b[s] == 1;
        //advance either to openPartOffset or numberOfSchemaFields
        s += 1;
        if (isExpanded) {
            openPartOffset = start + AInt32SerializerDeserializer.getInt(b, s);
            //advance to numberOfSchemaFields
            s += 4;
        }
    }
    try {
        if (numberOfSchemaFields > 0) {
            //advance to nullBitMap if hasOptionalFields, or fieldOffsets
            s += 4;
            int nullBitMapOffset = 0;
            boolean hasOptionalFields = NonTaggedFormatUtil.hasOptionalField(inputRecType);
            if (hasOptionalFields) {
                nullBitMapOffset = s;
                offsetArrayOffset = s + (this.numberOfSchemaFields % 4 == 0 ? numberOfSchemaFields / 4 : numberOfSchemaFields / 4 + 1);
            } else {
                offsetArrayOffset = s;
            }
            for (int i = 0; i < numberOfSchemaFields; i++) {
                fieldOffsets[i] = AInt32SerializerDeserializer.getInt(b, offsetArrayOffset) + recordOffset;
                offsetArrayOffset += 4;
            }
            for (int fieldNumber = 0; fieldNumber < numberOfSchemaFields; fieldNumber++) {
                if (hasOptionalFields) {
                    byte b1 = b[nullBitMapOffset + fieldNumber / 4];
                    int p = 1 << (7 - 2 * (fieldNumber % 4));
                    if ((b1 & p) == 0) {
                        // set null value (including type tag inside)
                        fieldValues.add(nullReference);
                        continue;
                    }
                    p = 1 << (7 - 2 * (fieldNumber % 4) - 1);
                    if ((b1 & p) == 0) {
                        // set missing value (including type tag inside)
                        fieldValues.add(missingReference);
                        continue;
                    }
                }
                IAType[] fieldTypes = inputRecType.getFieldTypes();
                int fieldValueLength = 0;
                IAType fieldType = fieldTypes[fieldNumber];
                if (fieldTypes[fieldNumber].getTypeTag() == ATypeTag.UNION) {
                    if (((AUnionType) fieldTypes[fieldNumber]).isUnknownableType()) {
                        fieldType = ((AUnionType) fieldTypes[fieldNumber]).getActualType();
                        typeTag = fieldType.getTypeTag();
                        fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(b, fieldOffsets[fieldNumber], typeTag, false);
                    }
                } else {
                    typeTag = fieldTypes[fieldNumber].getTypeTag();
                    fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(b, fieldOffsets[fieldNumber], typeTag, false);
                }
                // set field value (including the type tag)
                int fstart = dataBos.size();
                dataDos.writeByte(typeTag.serialize());
                dataDos.write(b, fieldOffsets[fieldNumber], fieldValueLength);
                int fend = dataBos.size();
                IVisitablePointable fieldValue = allocator.allocateFieldValue(fieldType);
                fieldValue.set(dataBos.getByteArray(), fstart, fend - fstart);
                fieldValues.add(fieldValue);
            }
        }
        if (isExpanded) {
            int numberOfOpenFields = AInt32SerializerDeserializer.getInt(b, openPartOffset);
            int fieldOffset = openPartOffset + 4 + (8 * numberOfOpenFields);
            for (int i = 0; i < numberOfOpenFields; i++) {
                // set the field name (including a type tag, which is
                // astring)
                int fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(b, fieldOffset, ATypeTag.STRING, false);
                int fnstart = dataBos.size();
                dataDos.writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG);
                dataDos.write(b, fieldOffset, fieldValueLength);
                int fnend = dataBos.size();
                IVisitablePointable fieldName = allocator.allocateEmpty();
                fieldName.set(dataBos.getByteArray(), fnstart, fnend - fnstart);
                fieldNames.add(fieldName);
                fieldOffset += fieldValueLength;
                // set the field type tag
                IVisitablePointable fieldTypeTag = allocator.allocateEmpty();
                fieldTypeTag.set(b, fieldOffset, 1);
                fieldTypeTags.add(fieldTypeTag);
                typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(b[fieldOffset]);
                // set the field value (already including type tag)
                fieldValueLength = NonTaggedFormatUtil.getFieldValueLength(b, fieldOffset, typeTag, true) + 1;
                // allocate
                IVisitablePointable fieldValueAccessor = allocator.allocateFieldValue(typeTag, b, fieldOffset + 1);
                fieldValueAccessor.set(b, fieldOffset, fieldValueLength);
                fieldValues.add(fieldValueAccessor);
                fieldOffset += fieldValueLength;
            }
        }
    } catch (Exception e) {
        throw new IllegalStateException(e);
    }
}

Example 10 with AUnionType

use of org.apache.asterix.om.types.AUnionType in project asterixdb by apache.

the class TweetParser method writeRecord.

public void writeRecord(JsonNode obj, DataOutput out, ARecordType curRecType) throws IOException {
    IAType[] curTypes = null;
    String[] curFNames = null;
    int fieldN;
    int attrIdx;
    int expectedFieldsCount = 0;
    ArrayBackedValueStorage fieldValueBuffer = getTempBuffer();
    ArrayBackedValueStorage fieldNameBuffer = getTempBuffer();
    IARecordBuilder recBuilder = getRecordBuilder();
    if (curRecType != null) {
        curTypes = curRecType.getFieldTypes();
        curFNames = curRecType.getFieldNames();
        for (IAType curType : curTypes) {
            if (!(curType instanceof AUnionType)) {
                expectedFieldsCount++;
            }
        }
    }
    recBuilder.reset(curRecType);
    recBuilder.init();
    if (curRecType != null && !curRecType.isOpen()) {
        // closed record type
        fieldN = curFNames.length;
        for (int iter1 = 0; iter1 < fieldN; iter1++) {
            fieldValueBuffer.reset();
            DataOutput fieldOutput = fieldValueBuffer.getDataOutput();
            if (obj.get(curFNames[iter1]).isNull() && !(curTypes[iter1] instanceof AUnionType)) {
                if (curRecType.isClosedField(curFNames[iter1])) {
                    throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL, curFNames[iter1]);
                } else {
                    continue;
                }
            } else {
                if (writeField(obj.get(curFNames[iter1]), curTypes[iter1], fieldOutput)) {
                    recBuilder.addField(iter1, fieldValueBuffer);
                }
            }
        }
    } else {
        //open record type
        int closedFieldCount = 0;
        IAType curFieldType = null;
        String attrName;
        Iterator<String> iter = obj.fieldNames();
        while (iter.hasNext()) {
            attrName = iter.next();
            if (obj.get(attrName) == null || obj.get(attrName).isNull() || obj.size() == 0) {
                continue;
            }
            attrIdx = checkAttrNameIdx(curFNames, attrName);
            if (curRecType != null) {
                curFieldType = curRecType.getFieldType(attrName);
            }
            fieldValueBuffer.reset();
            fieldNameBuffer.reset();
            DataOutput fieldOutput = fieldValueBuffer.getDataOutput();
            if (writeField(obj.get(attrName), curFieldType, fieldOutput)) {
                if (attrIdx == -1) {
                    aString.setValue(attrName);
                    stringSerde.serialize(aString, fieldNameBuffer.getDataOutput());
                    recBuilder.addField(fieldNameBuffer, fieldValueBuffer);
                } else {
                    recBuilder.addField(attrIdx, fieldValueBuffer);
                    closedFieldCount++;
                }
            }
        }
        if (curRecType != null && closedFieldCount < expectedFieldsCount) {
            throw new HyracksDataException("Non-null field is null");
        }
    }
    recBuilder.write(out, true);
}