Search in sources :

Example 16 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project presto by prestodb.

the class RcFileTester method decodeRecordReaderValue.

private static Object decodeRecordReaderValue(Type type, Object actualValue) {
    if (actualValue instanceof LazyPrimitive) {
        actualValue = ((LazyPrimitive<?, ?>) actualValue).getWritableObject();
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    } else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    } else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    } else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    } else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    } else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    } else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    } else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    } else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    } else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    } else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    } else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = new SqlTimestamp((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), UTC_KEY);
    } else if (actualValue instanceof StructObject) {
        StructObject structObject = (StructObject) actualValue;
        actualValue = decodeRecordReaderStruct(type, structObject.getFieldsAsList());
    } else if (actualValue instanceof LazyBinaryArray) {
        actualValue = decodeRecordReaderList(type, ((LazyBinaryArray) actualValue).getList());
    } else if (actualValue instanceof LazyBinaryMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyBinaryMap) actualValue).getMap());
    } else if (actualValue instanceof LazyArray) {
        actualValue = decodeRecordReaderList(type, ((LazyArray) actualValue).getList());
    } else if (actualValue instanceof LazyMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyMap) actualValue).getMap());
    } else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    return actualValue;
}
Also used : SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) LazyBinaryArray(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) LazyPrimitive(org.apache.hadoop.hive.serde2.lazy.LazyPrimitive) StructObject(org.apache.hadoop.hive.serde2.StructObject) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) BytesWritable(org.apache.hadoop.io.BytesWritable) SqlDecimal(com.facebook.presto.spi.type.SqlDecimal) Text(org.apache.hadoop.io.Text) FloatWritable(org.apache.hadoop.io.FloatWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) SqlDate(com.facebook.presto.spi.type.SqlDate) DecimalType(com.facebook.presto.spi.type.DecimalType) BigInteger(java.math.BigInteger) LazyArray(org.apache.hadoop.hive.serde2.lazy.LazyArray) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap)

Example 17 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project cdap by caskdata.

the class ObjectSerializer method fromLazyObject.

private Object fromLazyObject(TypeInfo type, Object data) {
    if (data == null) {
        return null;
    }
    switch(type.getCategory()) {
        case PRIMITIVE:
            Writable writable = ((LazyPrimitive) data).getWritableObject();
            return fromWritable(writable);
        case LIST:
            ListTypeInfo listType = (ListTypeInfo) type;
            TypeInfo listElementType = listType.getListElementTypeInfo();
            List<Object> list = ((LazyArray) data).getList();
            if (list.isEmpty()) {
                return ImmutableList.of();
            }
            Object[] arrayContent = new Object[list.size()];
            for (int i = 0; i < arrayContent.length; i++) {
                arrayContent[i] = fromLazyObject(listElementType, list.get(i));
            }
            return arrayContent;
        case MAP:
            MapTypeInfo mapType = (MapTypeInfo) type;
            Map<Object, Object> mapContent = Maps.newConcurrentMap();
            Map<Object, Object> map = ((LazyMap) data).getMap();
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                mapContent.put(fromLazyObject(mapType.getMapKeyTypeInfo(), entry.getKey()), fromLazyObject(mapType.getMapValueTypeInfo(), entry.getValue()));
            }
            return mapContent;
        case STRUCT:
            StructTypeInfo structType = (StructTypeInfo) type;
            List<TypeInfo> info = structType.getAllStructFieldTypeInfos();
            List<String> names = structType.getAllStructFieldNames();
            Map<String, Object> structMap = Maps.newConcurrentMap();
            List<Object> struct = ((LazyStruct) data).getFieldsAsList();
            for (int structIndex = 0; structIndex < info.size(); structIndex++) {
                structMap.put(names.get(structIndex), fromLazyObject(info.get(structIndex), struct.get(structIndex)));
            }
            return structMap;
        case UNION:
            throw new UnsupportedOperationException("union not yet supported");
        default:
            return data.toString();
    }
}
Also used : LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) ByteWritable(org.apache.hadoop.hive.serde2.io.ByteWritable) NullWritable(org.apache.hadoop.io.NullWritable) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) ShortWritable(org.apache.hadoop.io.ShortWritable) IntWritable(org.apache.hadoop.io.IntWritable) HiveBaseCharWritable(org.apache.hadoop.hive.serde2.io.HiveBaseCharWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) LazyPrimitive(org.apache.hadoop.hive.serde2.lazy.LazyPrimitive) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) LazyArray(org.apache.hadoop.hive.serde2.lazy.LazyArray) HashMap(java.util.HashMap) Map(java.util.Map) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) LazyStruct(org.apache.hadoop.hive.serde2.lazy.LazyStruct)

Example 18 with LazyPrimitive

use of org.apache.hadoop.hive.serde2.lazy.LazyPrimitive in project cdap by caskdata.

the class ObjectSerializer method serialize.

public Writable serialize(Object o, ObjectInspector objectInspector) {
    //overwrite field names (as they get lost by Hive)
    StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(objectInspector);
    structTypeInfo.setAllStructFieldNames(columnNames);
    List<TypeInfo> info = structTypeInfo.getAllStructFieldTypeInfos();
    List<String> names = structTypeInfo.getAllStructFieldNames();
    Map<String, Object> recordMap = new HashMap<>();
    List<Object> recordObjects = ((StructObjectInspector) objectInspector).getStructFieldsDataAsList(o);
    for (int structIndex = 0; structIndex < info.size(); structIndex++) {
        Object obj = recordObjects.get(structIndex);
        TypeInfo objType = info.get(structIndex);
        if (obj instanceof LazyNonPrimitive || obj instanceof LazyPrimitive) {
            // In case the SerDe that deserialized the object is the one of a native table
            recordMap.put(names.get(structIndex), fromLazyObject(objType, obj));
        } else if (obj instanceof Writable) {
            // Native tables sometimes introduce primitive Writable objects at this point
            recordMap.put(names.get(structIndex), fromWritable((Writable) obj));
        } else {
            // In case the deserializer is the DatasetSerDe
            recordMap.put(names.get(structIndex), serialize(obj, objType));
        }
    }
    // TODO Improve serialization logic - CDAP-11
    return new Text(GSON.toJson(recordMap));
}
Also used : HashMap(java.util.HashMap) ByteWritable(org.apache.hadoop.hive.serde2.io.ByteWritable) NullWritable(org.apache.hadoop.io.NullWritable) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) ShortWritable(org.apache.hadoop.io.ShortWritable) IntWritable(org.apache.hadoop.io.IntWritable) HiveBaseCharWritable(org.apache.hadoop.hive.serde2.io.HiveBaseCharWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) Text(org.apache.hadoop.io.Text) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) LazyPrimitive(org.apache.hadoop.hive.serde2.lazy.LazyPrimitive) LazyNonPrimitive(org.apache.hadoop.hive.serde2.lazy.LazyNonPrimitive) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)

Aggregations

LazyPrimitive (org.apache.hadoop.hive.serde2.lazy.LazyPrimitive)12 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)9 Text (org.apache.hadoop.io.Text)8 TypeInfo (org.apache.hadoop.hive.serde2.typeinfo.TypeInfo)5 BooleanWritable (org.apache.hadoop.io.BooleanWritable)5 BytesWritable (org.apache.hadoop.io.BytesWritable)5 FloatWritable (org.apache.hadoop.io.FloatWritable)5 IntWritable (org.apache.hadoop.io.IntWritable)5 LongWritable (org.apache.hadoop.io.LongWritable)5 Put (org.apache.hadoop.hbase.client.Put)4 ByteWritable (org.apache.hadoop.hive.serde2.io.ByteWritable)4 LazyMap (org.apache.hadoop.hive.serde2.lazy.LazyMap)4 LazyObject (org.apache.hadoop.hive.serde2.lazy.LazyObject)4 LazyMapObjectInspector (org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector)4 Writable (org.apache.hadoop.io.Writable)4 ArrayList (java.util.ArrayList)3 Map (java.util.Map)3 DateWritable (org.apache.hadoop.hive.serde2.io.DateWritable)3 HiveDecimalWritable (org.apache.hadoop.hive.serde2.io.HiveDecimalWritable)3 ShortWritable (org.apache.hadoop.hive.serde2.io.ShortWritable)3