Search in sources :

Example 51 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class AvroSerializer method serializeList.

private Object serializeList(ListTypeInfo typeInfo, ListObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException {
    List<?> list = fieldOI.getList(structFieldData);
    List<Object> deserialized = new GenericData.Array<Object>(list.size(), schema);
    TypeInfo listElementTypeInfo = typeInfo.getListElementTypeInfo();
    ObjectInspector listElementObjectInspector = fieldOI.getListElementObjectInspector();
    Schema elementType = schema.getElementType();
    for (int i = 0; i < list.size(); i++) {
        deserialized.add(i, serialize(listElementTypeInfo, listElementObjectInspector, list.get(i), elementType));
    }
    return deserialized;
}
Also used : GenericArray(org.apache.avro.generic.GenericArray) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) TimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector) DateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector) Schema(org.apache.avro.Schema) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo)

Example 52 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project mongo-hadoop by mongodb.

the class BSONSerDeTest method testList.

@Test
public void testList() throws SerDeException {
    String columnNames = "a";
    String columnTypes = "array<string>";
    String inner = "inside";
    ArrayList<String> value = new ArrayList<String>();
    value.add(inner);
    BasicBSONList b = new BasicBSONList();
    b.add(inner);
    BSONSerDe serde = new BSONSerDe();
    Object result = helpDeserialize(serde, columnNames, columnTypes, b);
    assertThat(value.toArray(), equalTo(result));
    // Since objectid is currently taken to be a string
    ObjectInspector innerInspector = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(String.class);
    ListObjectInspector listInspector = ObjectInspectorFactory.getStandardListObjectInspector(innerInspector);
    BasicBSONObject bObject = new BasicBSONObject();
    Object serialized = helpSerialize(columnNames, listInspector, bObject, value, serde);
    assertThat(new BSONWritable(bObject), equalTo(serialized));
}
Also used : BSONWritable(com.mongodb.hadoop.io.BSONWritable) BasicBSONObject(org.bson.BasicBSONObject) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) BasicBSONList(org.bson.types.BasicBSONList) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) ArrayList(java.util.ArrayList) BasicBSONObject(org.bson.BasicBSONObject) Test(org.junit.Test)

Example 53 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project mongo-hadoop by mongodb.

the class BSONSerDe method serializeList.

private Object serializeList(final Object obj, final ListObjectInspector oi, final String ext) {
    BasicBSONList list = new BasicBSONList();
    List<?> field = oi.getList(obj);
    if (field == null) {
        return list;
    }
    ObjectInspector elemOI = oi.getListElementObjectInspector();
    for (Object elem : field) {
        list.add(serializeObject(elem, elemOI, ext));
    }
    return list;
}
Also used : ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) BasicBSONList(org.bson.types.BasicBSONList) BasicBSONObject(org.bson.BasicBSONObject) BSONObject(org.bson.BSONObject)

Example 54 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project asterixdb by apache.

the class HiveRecordParser method parseUnorderedList.

private void parseUnorderedList(AUnorderedListType uoltype, Object obj, ListObjectInspector oi) throws IOException {
    UnorderedListBuilder unorderedListBuilder = getUnorderedListBuilder();
    IAType itemType = null;
    if (uoltype != null)
        itemType = uoltype.getItemType();
    byte tagByte = itemType.getTypeTag().serialize();
    unorderedListBuilder.reset(uoltype);
    int n = oi.getListLength(obj);
    for (int i = 0; i < n; i++) {
        Object element = oi.getListElement(obj, i);
        ObjectInspector eoi = oi.getListElementObjectInspector();
        if (element == null) {
            throw new RuntimeDataException(ErrorCode.PARSER_HIVE_NULL_VALUE_IN_LIST);
        }
        listItemBuffer.reset();
        final DataOutput dataOutput = listItemBuffer.getDataOutput();
        dataOutput.writeByte(tagByte);
        parseItem(itemType, element, eoi, dataOutput, true);
        unorderedListBuilder.addItem(listItemBuffer);
    }
    unorderedListBuilder.write(fieldValueBuffer.getDataOutput(), true);
}
Also used : DataOutput(java.io.DataOutput) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) IntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector) BooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector) ShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) LongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) FloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector) StringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector) ByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector) DoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector) TimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector) UnorderedListBuilder(org.apache.asterix.builders.UnorderedListBuilder) IAType(org.apache.asterix.om.types.IAType) RuntimeDataException(org.apache.asterix.common.exceptions.RuntimeDataException)

Example 55 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class DataWritableWriter method createWriter.

/**
 * Creates a writer for the specific object inspector. The returned writer will be used
 * to call Parquet API for the specific data type.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the type schema.
 * @return A ParquetWriter object used to call the Parquet API fo the specific data type.
 */
private DataWriter createWriter(ObjectInspector inspector, Type type) {
    if (type.isPrimitive()) {
        checkInspectorCategory(inspector, ObjectInspector.Category.PRIMITIVE);
        PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) inspector;
        switch(primitiveInspector.getPrimitiveCategory()) {
            case BOOLEAN:
                return new BooleanDataWriter((BooleanObjectInspector) inspector);
            case BYTE:
                return new ByteDataWriter((ByteObjectInspector) inspector);
            case SHORT:
                return new ShortDataWriter((ShortObjectInspector) inspector);
            case INT:
                return new IntDataWriter((IntObjectInspector) inspector);
            case LONG:
                return new LongDataWriter((LongObjectInspector) inspector);
            case FLOAT:
                return new FloatDataWriter((FloatObjectInspector) inspector);
            case DOUBLE:
                return new DoubleDataWriter((DoubleObjectInspector) inspector);
            case STRING:
                return new StringDataWriter((StringObjectInspector) inspector);
            case CHAR:
                return new CharDataWriter((HiveCharObjectInspector) inspector);
            case VARCHAR:
                return new VarcharDataWriter((HiveVarcharObjectInspector) inspector);
            case BINARY:
                return new BinaryDataWriter((BinaryObjectInspector) inspector);
            case TIMESTAMP:
                return new TimestampDataWriter((TimestampObjectInspector) inspector);
            case DECIMAL:
                return new DecimalDataWriter((HiveDecimalObjectInspector) inspector);
            case DATE:
                return new DateDataWriter((DateObjectInspector) inspector);
            default:
                throw new IllegalArgumentException("Unsupported primitive data type: " + primitiveInspector.getPrimitiveCategory());
        }
    } else {
        GroupType groupType = type.asGroupType();
        OriginalType originalType = type.getOriginalType();
        if (originalType != null && originalType.equals(OriginalType.LIST)) {
            checkInspectorCategory(inspector, ObjectInspector.Category.LIST);
            return new ListDataWriter((ListObjectInspector) inspector, groupType);
        } else if (originalType != null && originalType.equals(OriginalType.MAP)) {
            checkInspectorCategory(inspector, ObjectInspector.Category.MAP);
            return new MapDataWriter((MapObjectInspector) inspector, groupType);
        } else {
            checkInspectorCategory(inspector, ObjectInspector.Category.STRUCT);
            return new StructDataWriter((StructObjectInspector) inspector, groupType);
        }
    }
}
Also used : OriginalType(org.apache.parquet.schema.OriginalType) GroupType(org.apache.parquet.schema.GroupType) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)

Aggregations

ListObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector)57 ObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)43 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)42 MapObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector)35 PrimitiveObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector)29 ArrayList (java.util.ArrayList)28 List (java.util.List)18 LongObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector)18 TimestampObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector)18 DoubleObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector)17 IntObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector)17 ShortObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector)17 ByteObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector)16 FloatObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector)16 StringObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector)16 StructField (org.apache.hadoop.hive.serde2.objectinspector.StructField)15 BinaryObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector)15 BooleanObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector)15 HiveDecimalObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector)15 DateObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector)14