Search in sources :

Example 91 with HiveDecimal

use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.

the class BinarySortableSerDe method serialize.

static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, boolean invert, byte nullMarker, byte notNullMarker) throws SerDeException {
    // Is this field a null?
    if (o == null) {
        writeByte(buffer, nullMarker, invert);
        return;
    }
    // This field is not a null.
    writeByte(buffer, notNullMarker, invert);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        {
                            return;
                        }
                    case BOOLEAN:
                        {
                            boolean v = ((BooleanObjectInspector) poi).get(o);
                            writeByte(buffer, (byte) (v ? 2 : 1), invert);
                            return;
                        }
                    case BYTE:
                        {
                            ByteObjectInspector boi = (ByteObjectInspector) poi;
                            byte v = boi.get(o);
                            writeByte(buffer, (byte) (v ^ 0x80), invert);
                            return;
                        }
                    case SHORT:
                        {
                            ShortObjectInspector spoi = (ShortObjectInspector) poi;
                            short v = spoi.get(o);
                            serializeShort(buffer, v, invert);
                            return;
                        }
                    case INT:
                        {
                            IntObjectInspector ioi = (IntObjectInspector) poi;
                            int v = ioi.get(o);
                            serializeInt(buffer, v, invert);
                            return;
                        }
                    case LONG:
                        {
                            LongObjectInspector loi = (LongObjectInspector) poi;
                            long v = loi.get(o);
                            serializeLong(buffer, v, invert);
                            return;
                        }
                    case FLOAT:
                        {
                            FloatObjectInspector foi = (FloatObjectInspector) poi;
                            serializeFloat(buffer, foi.get(o), invert);
                            return;
                        }
                    case DOUBLE:
                        {
                            DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                            serializeDouble(buffer, doi.get(o), invert);
                            return;
                        }
                    case STRING:
                        {
                            StringObjectInspector soi = (StringObjectInspector) poi;
                            Text t = soi.getPrimitiveWritableObject(o);
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case CHAR:
                        {
                            HiveCharObjectInspector hcoi = (HiveCharObjectInspector) poi;
                            HiveCharWritable hc = hcoi.getPrimitiveWritableObject(o);
                            // Trailing space should ignored for char comparisons.
                            // So write stripped values for this SerDe.
                            Text t = hc.getStrippedValue();
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case VARCHAR:
                        {
                            HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector) poi;
                            HiveVarcharWritable hc = hcoi.getPrimitiveWritableObject(o);
                            // use varchar's text field directly
                            Text t = hc.getTextValue();
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case BINARY:
                        {
                            BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
                            BytesWritable ba = baoi.getPrimitiveWritableObject(o);
                            byte[] toSer = new byte[ba.getLength()];
                            System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
                            serializeBytes(buffer, toSer, ba.getLength(), invert);
                            return;
                        }
                    case DATE:
                        {
                            DateObjectInspector doi = (DateObjectInspector) poi;
                            int v = doi.getPrimitiveWritableObject(o).getDays();
                            serializeInt(buffer, v, invert);
                            return;
                        }
                    case TIMESTAMP:
                        {
                            TimestampObjectInspector toi = (TimestampObjectInspector) poi;
                            TimestampWritable t = toi.getPrimitiveWritableObject(o);
                            serializeTimestampWritable(buffer, t, invert);
                            return;
                        }
                    case TIMESTAMPLOCALTZ:
                        {
                            TimestampLocalTZObjectInspector toi = (TimestampLocalTZObjectInspector) poi;
                            TimestampLocalTZWritable t = toi.getPrimitiveWritableObject(o);
                            serializeTimestampTZWritable(buffer, t, invert);
                            return;
                        }
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonthObjectInspector ioi = (HiveIntervalYearMonthObjectInspector) poi;
                            HiveIntervalYearMonth intervalYearMonth = ioi.getPrimitiveJavaObject(o);
                            serializeHiveIntervalYearMonth(buffer, intervalYearMonth, invert);
                            return;
                        }
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTimeObjectInspector ioi = (HiveIntervalDayTimeObjectInspector) poi;
                            HiveIntervalDayTime intervalDayTime = ioi.getPrimitiveJavaObject(o);
                            serializeHiveIntervalDayTime(buffer, intervalDayTime, invert);
                            return;
                        }
                    case DECIMAL:
                        {
                            HiveDecimalObjectInspector boi = (HiveDecimalObjectInspector) poi;
                            HiveDecimal dec = boi.getPrimitiveJavaObject(o);
                            serializeHiveDecimal(buffer, dec, invert);
                            return;
                        }
                    default:
                        {
                            throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) oi;
                ObjectInspector eoi = loi.getListElementObjectInspector();
                // \1 followed by each element
                int size = loi.getListLength(o);
                for (int eid = 0; eid < size; eid++) {
                    writeByte(buffer, (byte) 1, invert);
                    serialize(buffer, loi.getListElement(o, eid), eoi, invert, nullMarker, notNullMarker);
                }
                // and \0 to terminate
                writeByte(buffer, (byte) 0, invert);
                return;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) oi;
                ObjectInspector koi = moi.getMapKeyObjectInspector();
                ObjectInspector voi = moi.getMapValueObjectInspector();
                // \1 followed by each key and then each value
                Map<?, ?> map = moi.getMap(o);
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    writeByte(buffer, (byte) 1, invert);
                    serialize(buffer, entry.getKey(), koi, invert, nullMarker, notNullMarker);
                    serialize(buffer, entry.getValue(), voi, invert, nullMarker, notNullMarker);
                }
                // and \0 to terminate
                writeByte(buffer, (byte) 0, invert);
                return;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) oi;
                List<? extends StructField> fields = soi.getAllStructFieldRefs();
                for (int i = 0; i < fields.size(); i++) {
                    serialize(buffer, soi.getStructFieldData(o, fields.get(i)), fields.get(i).getFieldObjectInspector(), invert, nullMarker, notNullMarker);
                }
                return;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) oi;
                byte tag = uoi.getTag(o);
                writeByte(buffer, tag, invert);
                serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert, nullMarker, notNullMarker);
                return;
            }
        default:
            {
                throw new RuntimeException("Unrecognized type: " + oi.getCategory());
            }
    }
}
Also used : LongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector) DateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector) IntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector) BinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) StringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector) FloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector) ByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector) TimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) ShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) HiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector) List(java.util.List) ArrayList(java.util.ArrayList) HiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector) HiveIntervalDayTime(org.apache.hadoop.hive.common.type.HiveIntervalDayTime) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) HiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector) BooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector) ShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector) HiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) FloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector) StringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector) HiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector) DateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) TimestampLocalTZObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector) HiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector) HiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector) IntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) LongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector) BinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector) ByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector) DoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector) TimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector) HiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) HiveVarcharWritable(org.apache.hadoop.hive.serde2.io.HiveVarcharWritable) TimestampLocalTZObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector) Text(org.apache.hadoop.io.Text) BytesWritable(org.apache.hadoop.io.BytesWritable) DoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector) HiveIntervalYearMonth(org.apache.hadoop.hive.common.type.HiveIntervalYearMonth) HiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector) HiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector) TimestampLocalTZWritable(org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) BooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector) Map(java.util.Map) HashMap(java.util.HashMap) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)

Example 92 with HiveDecimal

use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.

the class VerifyFast method serializeWrite.

public static void serializeWrite(SerializeWrite serializeWrite, TypeInfo typeInfo, Object object) throws IOException {
    if (object == null) {
        serializeWrite.writeNull();
        return;
    }
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                switch(primitiveTypeInfo.getPrimitiveCategory()) {
                    case BOOLEAN:
                        {
                            boolean value = ((BooleanWritable) object).get();
                            serializeWrite.writeBoolean(value);
                        }
                        break;
                    case BYTE:
                        {
                            byte value = ((ByteWritable) object).get();
                            serializeWrite.writeByte(value);
                        }
                        break;
                    case SHORT:
                        {
                            short value = ((ShortWritable) object).get();
                            serializeWrite.writeShort(value);
                        }
                        break;
                    case INT:
                        {
                            int value = ((IntWritable) object).get();
                            serializeWrite.writeInt(value);
                        }
                        break;
                    case LONG:
                        {
                            long value = ((LongWritable) object).get();
                            serializeWrite.writeLong(value);
                        }
                        break;
                    case FLOAT:
                        {
                            float value = ((FloatWritable) object).get();
                            serializeWrite.writeFloat(value);
                        }
                        break;
                    case DOUBLE:
                        {
                            double value = ((DoubleWritable) object).get();
                            serializeWrite.writeDouble(value);
                        }
                        break;
                    case STRING:
                        {
                            Text value = (Text) object;
                            byte[] stringBytes = value.getBytes();
                            int stringLength = stringBytes.length;
                            serializeWrite.writeString(stringBytes, 0, stringLength);
                        }
                        break;
                    case CHAR:
                        {
                            HiveChar value = ((HiveCharWritable) object).getHiveChar();
                            serializeWrite.writeHiveChar(value);
                        }
                        break;
                    case VARCHAR:
                        {
                            HiveVarchar value = ((HiveVarcharWritable) object).getHiveVarchar();
                            serializeWrite.writeHiveVarchar(value);
                        }
                        break;
                    case DECIMAL:
                        {
                            HiveDecimal value = ((HiveDecimalWritable) object).getHiveDecimal();
                            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                            serializeWrite.writeHiveDecimal(value, decTypeInfo.scale());
                        }
                        break;
                    case DATE:
                        {
                            Date value = ((DateWritable) object).get();
                            serializeWrite.writeDate(value);
                        }
                        break;
                    case TIMESTAMP:
                        {
                            Timestamp value = ((TimestampWritable) object).getTimestamp();
                            serializeWrite.writeTimestamp(value);
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonth value = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
                            serializeWrite.writeHiveIntervalYearMonth(value);
                        }
                        break;
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTime value = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
                            serializeWrite.writeHiveIntervalDayTime(value);
                        }
                        break;
                    case BINARY:
                        {
                            BytesWritable byteWritable = (BytesWritable) object;
                            byte[] binaryBytes = byteWritable.getBytes();
                            int length = byteWritable.getLength();
                            serializeWrite.writeBinary(binaryBytes, 0, length);
                        }
                        break;
                    default:
                        throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory().name());
                }
            }
            break;
        case LIST:
            {
                ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
                TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo();
                ArrayList<Object> elements = (ArrayList<Object>) object;
                serializeWrite.beginList(elements);
                boolean isFirst = true;
                for (Object elementObject : elements) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateList();
                    }
                    if (elementObject == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, elementTypeInfo, elementObject);
                    }
                }
                serializeWrite.finishList();
            }
            break;
        case MAP:
            {
                MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
                TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo();
                TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo();
                HashMap<Object, Object> hashMap = (HashMap<Object, Object>) object;
                serializeWrite.beginMap(hashMap);
                boolean isFirst = true;
                for (Entry<Object, Object> entry : hashMap.entrySet()) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateKeyValuePair();
                    }
                    if (entry.getKey() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, keyTypeInfo, entry.getKey());
                    }
                    serializeWrite.separateKey();
                    if (entry.getValue() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, valueTypeInfo, entry.getValue());
                    }
                }
                serializeWrite.finishMap();
            }
            break;
        case STRUCT:
            {
                StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
                ArrayList<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
                ArrayList<Object> fieldValues = (ArrayList<Object>) object;
                final int size = fieldValues.size();
                serializeWrite.beginStruct(fieldValues);
                boolean isFirst = true;
                for (int i = 0; i < size; i++) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateStruct();
                    }
                    serializeWrite(serializeWrite, fieldTypeInfos.get(i), fieldValues.get(i));
                }
                serializeWrite.finishStruct();
            }
            break;
        case UNION:
            {
                UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
                List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
                final int size = fieldTypeInfos.size();
                StandardUnion standardUnion = (StandardUnion) object;
                byte tag = standardUnion.getTag();
                serializeWrite.beginUnion(tag);
                serializeWrite(serializeWrite, fieldTypeInfos.get(tag), standardUnion.getObject());
                serializeWrite.finishUnion();
            }
            break;
        default:
            throw new Error("Unknown category " + typeInfo.getCategory().name());
    }
}
Also used : HashMap(java.util.HashMap) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) ArrayList(java.util.ArrayList) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) DoubleWritable(org.apache.hadoop.hive.serde2.io.DoubleWritable) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) Timestamp(java.sql.Timestamp) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) Entry(java.util.Map.Entry) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) ArrayList(java.util.ArrayList) List(java.util.List) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.hive.serde2.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) HiveIntervalDayTime(org.apache.hadoop.hive.common.type.HiveIntervalDayTime) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) HiveVarcharWritable(org.apache.hadoop.hive.serde2.io.HiveVarcharWritable) Text(org.apache.hadoop.io.Text) HiveIntervalDayTimeWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) HiveIntervalYearMonthWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) Date(java.sql.Date) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) FloatWritable(org.apache.hadoop.io.FloatWritable) HiveIntervalYearMonth(org.apache.hadoop.hive.common.type.HiveIntervalYearMonth) BooleanWritable(org.apache.hadoop.io.BooleanWritable) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) StandardUnion(org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector.StandardUnion) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo)

Example 93 with HiveDecimal

use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.

the class HiveDecimalWritable method getHiveDecimal.

/**
 * Get a HiveDecimal instance from the writable and constraint it with maximum precision/scale.
 * <p>
 * @param maxPrecision maximum precision
 * @param maxScale maximum scale
 * @return HiveDecimal instance
 */
@HiveDecimalWritableVersionV1
public HiveDecimal getHiveDecimal(int maxPrecision, int maxScale) {
    if (!isSet) {
        return null;
    }
    HiveDecimal dec = HiveDecimal.createFromFast(this);
    HiveDecimal result = HiveDecimal.enforcePrecisionScale(dec, maxPrecision, maxScale);
    return result;
}
Also used : FastHiveDecimal(org.apache.hadoop.hive.common.type.FastHiveDecimal) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal)

Example 94 with HiveDecimal

use of org.apache.hadoop.hive.common.type.HiveDecimal in project hive by apache.

the class VerifyLazy method lazyCompare.

public static boolean lazyCompare(TypeInfo typeInfo, Object lazyObject, Object expectedObject) {
    if (expectedObject == null) {
        if (lazyObject != null) {
            throw new RuntimeException("Expected object is null but object is not null " + lazyObject.toString() + " typeInfo " + typeInfo.toString());
        }
        return true;
    } else if (lazyObject == null) {
        throw new RuntimeException("Expected object is not null \"" + expectedObject.toString() + "\" typeInfo " + typeInfo.toString() + " but object is null");
    }
    if (lazyObject instanceof Writable) {
        if (!lazyObject.equals(expectedObject)) {
            throw new RuntimeException("Expected object " + expectedObject.toString() + " and actual object " + lazyObject.toString() + " is not equal typeInfo " + typeInfo.toString());
        }
        return true;
    }
    if (lazyObject instanceof LazyPrimitive) {
        Object primitiveObject = ((LazyPrimitive) lazyObject).getObject();
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
        switch(primitiveTypeInfo.getPrimitiveCategory()) {
            case BOOLEAN:
                {
                    if (!(primitiveObject instanceof LazyBoolean)) {
                        throw new RuntimeException("Expected LazyBoolean");
                    }
                    boolean value = ((LazyBoolean) primitiveObject).getWritableObject().get();
                    boolean expected = ((BooleanWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Boolean field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BYTE:
                {
                    if (!(primitiveObject instanceof LazyByte)) {
                        throw new RuntimeException("Expected LazyByte");
                    }
                    byte value = ((LazyByte) primitiveObject).getWritableObject().get();
                    byte expected = ((ByteWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                    }
                }
                break;
            case SHORT:
                {
                    if (!(primitiveObject instanceof LazyShort)) {
                        throw new RuntimeException("Expected LazyShort");
                    }
                    short value = ((LazyShort) primitiveObject).getWritableObject().get();
                    short expected = ((ShortWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Short field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INT:
                {
                    if (!(primitiveObject instanceof LazyInteger)) {
                        throw new RuntimeException("Expected LazyInteger");
                    }
                    int value = ((LazyInteger) primitiveObject).getWritableObject().get();
                    int expected = ((IntWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Int field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case LONG:
                {
                    if (!(primitiveObject instanceof LazyLong)) {
                        throw new RuntimeException("Expected LazyLong");
                    }
                    long value = ((LazyLong) primitiveObject).getWritableObject().get();
                    long expected = ((LongWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Long field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case FLOAT:
                {
                    if (!(primitiveObject instanceof LazyFloat)) {
                        throw new RuntimeException("Expected LazyFloat");
                    }
                    float value = ((LazyFloat) primitiveObject).getWritableObject().get();
                    float expected = ((FloatWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Float field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DOUBLE:
                {
                    if (!(primitiveObject instanceof LazyDouble)) {
                        throw new RuntimeException("Expected LazyDouble");
                    }
                    double value = ((LazyDouble) primitiveObject).getWritableObject().get();
                    double expected = ((DoubleWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Double field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case STRING:
                {
                    if (!(primitiveObject instanceof LazyString)) {
                        throw new RuntimeException("Text expected writable not Text");
                    }
                    Text value = ((LazyString) primitiveObject).getWritableObject();
                    Text expected = ((Text) expectedObject);
                    if (!value.equals(expected)) {
                        throw new RuntimeException("String field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case CHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveChar)) {
                        throw new RuntimeException("Expected LazyHiveChar");
                    }
                    HiveChar value = ((LazyHiveChar) primitiveObject).getWritableObject().getHiveChar();
                    HiveChar expected = ((HiveCharWritable) expectedObject).getHiveChar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveChar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case VARCHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveVarchar)) {
                        throw new RuntimeException("Expected LazyHiveVarchar");
                    }
                    HiveVarchar value = ((LazyHiveVarchar) primitiveObject).getWritableObject().getHiveVarchar();
                    HiveVarchar expected = ((HiveVarcharWritable) expectedObject).getHiveVarchar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveVarchar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case DECIMAL:
                {
                    if (!(primitiveObject instanceof LazyHiveDecimal)) {
                        throw new RuntimeException("Expected LazyDecimal");
                    }
                    HiveDecimal value = ((LazyHiveDecimal) primitiveObject).getWritableObject().getHiveDecimal();
                    HiveDecimal expected = ((HiveDecimalWritable) expectedObject).getHiveDecimal();
                    if (!value.equals(expected)) {
                        DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                        int precision = decimalTypeInfo.getPrecision();
                        int scale = decimalTypeInfo.getScale();
                        throw new RuntimeException("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                    }
                }
                break;
            case DATE:
                {
                    if (!(primitiveObject instanceof LazyDate)) {
                        throw new RuntimeException("Expected LazyDate");
                    }
                    Date value = ((LazyDate) primitiveObject).getWritableObject().get();
                    Date expected = ((DateWritable) expectedObject).get();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Date field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case TIMESTAMP:
                {
                    if (!(primitiveObject instanceof LazyTimestamp)) {
                        throw new RuntimeException("TimestampWritable expected writable not TimestampWritable");
                    }
                    Timestamp value = ((LazyTimestamp) primitiveObject).getWritableObject().getTimestamp();
                    Timestamp expected = ((TimestampWritable) expectedObject).getTimestamp();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Timestamp field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalYearMonth)) {
                        throw new RuntimeException("Expected LazyHiveIntervalYearMonth");
                    }
                    HiveIntervalYearMonth value = ((LazyHiveIntervalYearMonth) primitiveObject).getWritableObject().getHiveIntervalYearMonth();
                    HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) expectedObject).getHiveIntervalYearMonth();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalYearMonth field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalDayTime)) {
                        throw new RuntimeException("Expected writable LazyHiveIntervalDayTime");
                    }
                    HiveIntervalDayTime value = ((LazyHiveIntervalDayTime) primitiveObject).getWritableObject().getHiveIntervalDayTime();
                    HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) expectedObject).getHiveIntervalDayTime();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalDayTime field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BINARY:
                {
                    if (!(primitiveObject instanceof LazyBinary)) {
                        throw new RuntimeException("Expected LazyBinary");
                    }
                    BytesWritable bytesWritable = ((LazyBinary) primitiveObject).getWritableObject();
                    byte[] value = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                    BytesWritable bytesWritableExpected = (BytesWritable) expectedObject;
                    byte[] expected = Arrays.copyOfRange(bytesWritableExpected.getBytes(), 0, bytesWritableExpected.getLength());
                    if (value.length != expected.length) {
                        throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                    }
                    for (int b = 0; b < value.length; b++) {
                        if (value[b] != expected[b]) {
                            throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                        }
                    }
                }
                break;
            default:
                throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
        }
    } else if (lazyObject instanceof LazyArray) {
        LazyArray lazyArray = (LazyArray) lazyObject;
        List<Object> list = lazyArray.getList();
        List<Object> expectedList = (List<Object>) expectedObject;
        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")" + " elementTypeInfo " + listTypeInfo.getListElementTypeInfo().toString());
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (typeInfo instanceof ListTypeInfo) {
        List<Object> list;
        if (lazyObject instanceof LazyBinaryArray) {
            list = ((LazyBinaryArray) lazyObject).getList();
        } else {
            list = (List<Object>) lazyObject;
        }
        List<Object> expectedList = (List<Object>) expectedObject;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")");
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (lazyObject instanceof LazyMap) {
        LazyMap lazyMap = (LazyMap) lazyObject;
        Map<Object, Object> map = lazyMap.getMap();
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (typeInfo instanceof MapTypeInfo) {
        Map<Object, Object> map;
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        if (lazyObject instanceof LazyBinaryMap) {
            map = ((LazyBinaryMap) lazyObject).getMap();
        } else {
            map = (Map<Object, Object>) lazyObject;
        }
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (lazyObject instanceof LazyStruct) {
        LazyStruct lazyStruct = (LazyStruct) lazyObject;
        List<Object> fields = lazyStruct.getFieldsAsList();
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (typeInfo instanceof StructTypeInfo) {
        ArrayList<Object> fields;
        if (lazyObject instanceof LazyBinaryStruct) {
            fields = ((LazyBinaryStruct) lazyObject).getFieldsAsList();
        } else {
            fields = (ArrayList<Object>) lazyObject;
        }
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (lazyObject instanceof LazyUnion) {
        LazyUnion union = (LazyUnion) lazyObject;
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        return lazyCompareUnion(unionTypeInfo, union, expectedUnion);
    } else if (typeInfo instanceof UnionTypeInfo) {
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        if (lazyObject instanceof LazyBinaryUnion) {
            return lazyCompareUnion(unionTypeInfo, (LazyBinaryUnion) lazyObject, expectedUnion);
        } else {
            return lazyCompareUnion(unionTypeInfo, (UnionObject) lazyObject, expectedUnion);
        }
    } else {
        System.err.println("Not implemented " + typeInfo.getClass().getName());
    }
    return true;
}
Also used : StandardUnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector) ByteWritable(org.apache.hadoop.hive.serde2.io.ByteWritable) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) Writable(org.apache.hadoop.io.Writable) LongWritable(org.apache.hadoop.io.LongWritable) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) HiveIntervalYearMonthWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable) HiveIntervalDayTimeWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) DoubleWritable(org.apache.hadoop.hive.serde2.io.DoubleWritable) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) IntWritable(org.apache.hadoop.io.IntWritable) HiveVarcharWritable(org.apache.hadoop.hive.serde2.io.HiveVarcharWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) LazyBinaryArray(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) LazyBinaryStruct(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct) ArrayList(java.util.ArrayList) List(java.util.List) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) UnionObject(org.apache.hadoop.hive.serde2.objectinspector.UnionObject) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap) Map(java.util.Map) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) Timestamp(java.sql.Timestamp) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) LazyBinaryUnion(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUnion) HiveIntervalDayTime(org.apache.hadoop.hive.common.type.HiveIntervalDayTime) Text(org.apache.hadoop.io.Text) BytesWritable(org.apache.hadoop.io.BytesWritable) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) Date(java.sql.Date) HiveIntervalYearMonth(org.apache.hadoop.hive.common.type.HiveIntervalYearMonth) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo)

Aggregations

HiveDecimal (org.apache.hadoop.hive.common.type.HiveDecimal)94 Test (org.junit.Test)28 Timestamp (java.sql.Timestamp)24 HiveDecimalWritable (org.apache.hadoop.hive.serde2.io.HiveDecimalWritable)23 DecimalColumnVector (org.apache.hadoop.hive.ql.exec.vector.DecimalColumnVector)22 Text (org.apache.hadoop.io.Text)22 HiveChar (org.apache.hadoop.hive.common.type.HiveChar)21 PrimitiveTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo)21 HiveVarchar (org.apache.hadoop.hive.common.type.HiveVarchar)20 Date (java.sql.Date)19 VectorizedRowBatch (org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch)18 BytesWritable (org.apache.hadoop.io.BytesWritable)17 DecimalTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo)16 HiveIntervalYearMonth (org.apache.hadoop.hive.common.type.HiveIntervalYearMonth)15 DateWritable (org.apache.hadoop.hive.serde2.io.DateWritable)15 TimestampWritable (org.apache.hadoop.hive.serde2.io.TimestampWritable)15 HiveIntervalDayTime (org.apache.hadoop.hive.common.type.HiveIntervalDayTime)14 IntWritable (org.apache.hadoop.io.IntWritable)14 ByteWritable (org.apache.hadoop.hive.serde2.io.ByteWritable)13 DoubleWritable (org.apache.hadoop.hive.serde2.io.DoubleWritable)13