Examples with StructField org.apache.hadoop.hive.serde2.objectinspector.StructField used on opensource projects

Example 21 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class SerDeUtils method buildJSONString.

static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                if (o == null) {
                    sb.append(nullStr);
                } else {
                    switch(poi.getPrimitiveCategory()) {
                        case BOOLEAN:
                            {
                                boolean b = ((BooleanObjectInspector) poi).get(o);
                                sb.append(b ? "true" : "false");
                                break;
                            }
                        case BYTE:
                            {
                                sb.append(((ByteObjectInspector) poi).get(o));
                                break;
                            }
                        case SHORT:
                            {
                                sb.append(((ShortObjectInspector) poi).get(o));
                                break;
                            }
                        case INT:
                            {
                                sb.append(((IntObjectInspector) poi).get(o));
                                break;
                            }
                        case LONG:
                            {
                                sb.append(((LongObjectInspector) poi).get(o));
                                break;
                            }
                        case FLOAT:
                            {
                                sb.append(((FloatObjectInspector) poi).get(o));
                                break;
                            }
                        case DOUBLE:
                            {
                                sb.append(((DoubleObjectInspector) poi).get(o));
                                break;
                            }
                        case STRING:
                            {
                                sb.append('"');
                                sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
                                sb.append('"');
                                break;
                            }
                        case CHAR:
                            {
                                sb.append('"');
                                sb.append(escapeString(((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                                sb.append('"');
                                break;
                            }
                        case VARCHAR:
                            {
                                sb.append('"');
                                sb.append(escapeString(((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                                sb.append('"');
                                break;
                            }
                        case DATE:
                            {
                                sb.append('"');
                                sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
                                sb.append('"');
                                break;
                            }
                        case TIMESTAMP:
                            {
                                sb.append('"');
                                sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
                                sb.append('"');
                                break;
                            }
                        case BINARY:
                            {
                                BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                                Text txt = new Text();
                                txt.set(bw.getBytes(), 0, bw.getLength());
                                sb.append(txt.toString());
                                break;
                            }
                        case DECIMAL:
                            {
                                sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
                                break;
                            }
                        case INTERVAL_YEAR_MONTH:
                            {
                                sb.append(((HiveIntervalYearMonthObjectInspector) oi).getPrimitiveJavaObject(o));
                                break;
                            }
                        case INTERVAL_DAY_TIME:
                            {
                                sb.append(((HiveIntervalDayTimeObjectInspector) oi).getPrimitiveJavaObject(o));
                                break;
                            }
                        default:
                            throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
                    }
                }
                break;
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) oi;
                ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
                List<?> olist = loi.getList(o);
                if (olist == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACKET);
                    for (int i = 0; i < olist.size(); i++) {
                        if (i > 0) {
                            sb.append(COMMA);
                        }
                        buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
                    }
                    sb.append(RBRACKET);
                }
                break;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) oi;
                ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
                ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
                Map<?, ?> omap = moi.getMap(o);
                if (omap == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACE);
                    boolean first = true;
                    for (Object entry : omap.entrySet()) {
                        if (first) {
                            first = false;
                        } else {
                            sb.append(COMMA);
                        }
                        Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
                        buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
                        sb.append(COLON);
                        buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
                    }
                    sb.append(RBRACE);
                }
                break;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) oi;
                List<? extends StructField> structFields = soi.getAllStructFieldRefs();
                if (o == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACE);
                    for (int i = 0; i < structFields.size(); i++) {
                        if (i > 0) {
                            sb.append(COMMA);
                        }
                        sb.append(QUOTE);
                        sb.append(structFields.get(i).getFieldName());
                        sb.append(QUOTE);
                        sb.append(COLON);
                        buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector(), JSON_NULL);
                    }
                    sb.append(RBRACE);
                }
                break;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) oi;
                if (o == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACE);
                    sb.append(uoi.getTag(o));
                    sb.append(COLON);
                    buildJSONString(sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
                    sb.append(RBRACE);
                }
                break;
            }
        default:
            throw new RuntimeException("Unknown type in ObjectInspector!");
    }
}

Example 22 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class BinarySortableSerDe method serialize.

@Override
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
    output.reset();
    StructObjectInspector soi = (StructObjectInspector) objInspector;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();
    for (int i = 0; i < columnNames.size(); i++) {
        serialize(output, soi.getStructFieldData(obj, fields.get(i)), fields.get(i).getFieldObjectInspector(), columnSortOrderIsDesc[i], columnNullMarker[i], columnNotNullMarker[i]);
    }
    serializeBytesWritable.set(output.getData(), 0, output.getLength());
    return serializeBytesWritable;
}

Example 23 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class ColumnarSerDe method serialize.

/**
   * Serialize a row of data.
   *
   * @param obj
   *          The row object
   * @param objInspector
   *          The ObjectInspector for the row object
   * @return The serialized Writable object
   * @see AbstractSerDe#serialize(Object, ObjectInspector)
   */
@Override
public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
    if (objInspector.getCategory() != Category.STRUCT) {
        throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName());
    }
    // Prepare the field ObjectInspectors
    StructObjectInspector soi = (StructObjectInspector) objInspector;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();
    List<Object> list = soi.getStructFieldsDataAsList(obj);
    List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams.getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()).getAllStructFieldRefs() : null;
    try {
        // used for avoid extra byte copy
        serializeStream.reset();
        serializedSize = 0;
        int count = 0;
        // Serialize each field
        for (int i = 0; i < fields.size(); i++) {
            // Get the field objectInspector and the field object.
            ObjectInspector foi = fields.get(i).getFieldObjectInspector();
            Object f = (list == null ? null : list.get(i));
            if (declaredFields != null && i >= declaredFields.size()) {
                throw new SerDeException("Error: expecting " + declaredFields.size() + " but asking for field " + i + "\n" + "data=" + obj + "\n" + "tableType=" + serdeParams.getRowTypeInfo().toString() + "\n" + "dataType=" + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector));
            }
            // delimited way.
            if (!foi.getCategory().equals(Category.PRIMITIVE) && (declaredFields == null || declaredFields.get(i).getFieldObjectInspector().getCategory().equals(Category.PRIMITIVE))) {
                LazySimpleSerDe.serialize(serializeStream, SerDeUtils.getJSONString(f, foi), PrimitiveObjectInspectorFactory.javaStringObjectInspector, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape());
            } else {
                LazySimpleSerDe.serialize(serializeStream, f, foi, serdeParams.getSeparators(), 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape());
            }
            field[i].set(serializeStream.getData(), count, serializeStream.getLength() - count);
            count = serializeStream.getLength();
        }
        serializedSize = serializeStream.getLength();
        lastOperationSerialize = true;
        lastOperationDeserialize = false;
    } catch (IOException e) {
        throw new SerDeException(e);
    }
    return serializeCache;
}

Example 24 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class LazySimpleSerDe method doSerialize.

/**
   * Serialize a row of data.
   *
   * @param obj
   *          The row object
   * @param objInspector
   *          The ObjectInspector for the row object
   * @return The serialized Writable object
   * @throws IOException
   * @see AbstractSerDe#serialize(Object, ObjectInspector)
   */
@Override
public Writable doSerialize(Object obj, ObjectInspector objInspector) throws SerDeException {
    if (objInspector.getCategory() != Category.STRUCT) {
        throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objInspector.getTypeName());
    }
    // Prepare the field ObjectInspectors
    StructObjectInspector soi = (StructObjectInspector) objInspector;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();
    List<Object> list = soi.getStructFieldsDataAsList(obj);
    List<? extends StructField> declaredFields = (serdeParams.getRowTypeInfo() != null && ((StructTypeInfo) serdeParams.getRowTypeInfo()).getAllStructFieldNames().size() > 0) ? ((StructObjectInspector) getObjectInspector()).getAllStructFieldRefs() : null;
    serializeStream.reset();
    serializedSize = 0;
    // Serialize each field
    for (int i = 0; i < fields.size(); i++) {
        // Append the separator if needed.
        if (i > 0) {
            serializeStream.write(serdeParams.getSeparators()[0]);
        }
        // Get the field objectInspector and the field object.
        ObjectInspector foi = fields.get(i).getFieldObjectInspector();
        Object f = (list == null ? null : list.get(i));
        if (declaredFields != null && i >= declaredFields.size()) {
            throw new SerDeException("Error: expecting " + declaredFields.size() + " but asking for field " + i + "\n" + "data=" + obj + "\n" + "tableType=" + serdeParams.getRowTypeInfo().toString() + "\n" + "dataType=" + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector));
        }
        serializeField(serializeStream, f, foi, serdeParams);
    }
    // TODO: The copy of data is unnecessary, but there is no work-around
    // since we cannot directly set the private byte[] field inside Text.
    serializeCache.set(serializeStream.getData(), 0, serializeStream.getLength());
    serializedSize = serializeStream.getLength();
    lastOperationSerialize = true;
    lastOperationDeserialize = false;
    return serializeCache;
}

Example 25 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class TestAvroDeserializer method canDeserializeEnums.

// Enums are one of two types we fudge for Hive. Enums go in, Strings come out.
@Test
public void canDeserializeEnums() throws SerDeException, IOException {
    Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.ENUM_SCHEMA);
    GenericData.Record record = new GenericData.Record(s);
    record.put("baddies", new GenericData.EnumSymbol(s.getField("baddies").schema(), "DALEKS"));
    assertTrue(GENERIC_DATA.validate(s, record));
    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
    AvroDeserializer de = new AvroDeserializer();
    ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
    assertEquals(1, row.size());
    StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
    List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
    assertEquals(1, fieldRefs.size());
    StructField fieldRef = fieldRefs.get(0);
    assertEquals("baddies", fieldRef.getFieldName());
    Object theStringObject = oi.getStructFieldData(row, fieldRef);
    assertTrue(fieldRef.getFieldObjectInspector() instanceof StringObjectInspector);
    StringObjectInspector soi = (StringObjectInspector) fieldRef.getFieldObjectInspector();
    String finalValue = soi.getPrimitiveJavaObject(theStringObject);
    assertEquals("DALEKS", finalValue);
}