Search in sources :

Example 31 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project hive by apache.

the class SerDeUtils method buildJSONString.

static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                if (o == null) {
                    sb.append(nullStr);
                } else {
                    switch(poi.getPrimitiveCategory()) {
                        case BOOLEAN:
                            {
                                boolean b = ((BooleanObjectInspector) poi).get(o);
                                sb.append(b ? "true" : "false");
                                break;
                            }
                        case BYTE:
                            {
                                sb.append(((ByteObjectInspector) poi).get(o));
                                break;
                            }
                        case SHORT:
                            {
                                sb.append(((ShortObjectInspector) poi).get(o));
                                break;
                            }
                        case INT:
                            {
                                sb.append(((IntObjectInspector) poi).get(o));
                                break;
                            }
                        case LONG:
                            {
                                sb.append(((LongObjectInspector) poi).get(o));
                                break;
                            }
                        case FLOAT:
                            {
                                sb.append(((FloatObjectInspector) poi).get(o));
                                break;
                            }
                        case DOUBLE:
                            {
                                sb.append(((DoubleObjectInspector) poi).get(o));
                                break;
                            }
                        case STRING:
                            {
                                sb.append('"');
                                sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
                                sb.append('"');
                                break;
                            }
                        case CHAR:
                            {
                                sb.append('"');
                                sb.append(escapeString(((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                                sb.append('"');
                                break;
                            }
                        case VARCHAR:
                            {
                                sb.append('"');
                                sb.append(escapeString(((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
                                sb.append('"');
                                break;
                            }
                        case DATE:
                            {
                                sb.append('"');
                                sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
                                sb.append('"');
                                break;
                            }
                        case TIMESTAMP:
                            {
                                sb.append('"');
                                sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
                                sb.append('"');
                                break;
                            }
                        case TIMESTAMPLOCALTZ:
                            {
                                sb.append('"');
                                sb.append(((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o));
                                sb.append('"');
                                break;
                            }
                        case BINARY:
                            {
                                BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                                Text txt = new Text();
                                txt.set(bw.getBytes(), 0, bw.getLength());
                                sb.append(txt.toString());
                                break;
                            }
                        case DECIMAL:
                            {
                                sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
                                break;
                            }
                        case INTERVAL_YEAR_MONTH:
                            {
                                sb.append(((HiveIntervalYearMonthObjectInspector) oi).getPrimitiveJavaObject(o));
                                break;
                            }
                        case INTERVAL_DAY_TIME:
                            {
                                sb.append(((HiveIntervalDayTimeObjectInspector) oi).getPrimitiveJavaObject(o));
                                break;
                            }
                        default:
                            throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
                    }
                }
                break;
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) oi;
                ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
                List<?> olist = loi.getList(o);
                if (olist == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACKET);
                    for (int i = 0; i < olist.size(); i++) {
                        if (i > 0) {
                            sb.append(COMMA);
                        }
                        buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
                    }
                    sb.append(RBRACKET);
                }
                break;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) oi;
                ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
                ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
                Map<?, ?> omap = moi.getMap(o);
                if (omap == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACE);
                    boolean first = true;
                    for (Object entry : omap.entrySet()) {
                        if (first) {
                            first = false;
                        } else {
                            sb.append(COMMA);
                        }
                        Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
                        buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
                        sb.append(COLON);
                        buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
                    }
                    sb.append(RBRACE);
                }
                break;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) oi;
                List<? extends StructField> structFields = soi.getAllStructFieldRefs();
                if (o == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACE);
                    for (int i = 0; i < structFields.size(); i++) {
                        if (i > 0) {
                            sb.append(COMMA);
                        }
                        sb.append(QUOTE);
                        sb.append(structFields.get(i).getFieldName());
                        sb.append(QUOTE);
                        sb.append(COLON);
                        buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector(), JSON_NULL);
                    }
                    sb.append(RBRACE);
                }
                break;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) oi;
                if (o == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(LBRACE);
                    sb.append(uoi.getTag(o));
                    sb.append(COLON);
                    buildJSONString(sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
                    sb.append(RBRACE);
                }
                break;
            }
        default:
            throw new RuntimeException("Unknown type in ObjectInspector!");
    }
}
Also used : ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) TimestampLocalTZObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector) HiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) HiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector) IntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector) HiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector) BooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) ShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector) HiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) LongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector) BinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) FloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector) StringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector) HiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector) ByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector) DoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector) TimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector) DateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector) BinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector) BytesWritable(org.apache.hadoop.io.BytesWritable) Text(org.apache.hadoop.io.Text) StringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) PrimitiveObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector) List(java.util.List) BooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector) Map(java.util.Map) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)

Example 32 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project hive by apache.

the class AvroLazyObjectInspector method toLazyMapObject.

/**
 * Convert the given object to a lazy object using the given {@link ObjectInspector}
 *
 * @param obj Object to be converted to a {@link LazyObject}
 * @param objectInspector ObjectInspector used for the conversion
 * @return the created {@link LazyObject lazy object}
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
private Object toLazyMapObject(Object obj, ObjectInspector objectInspector) {
    if (obj == null) {
        return null;
    }
    // avro guarantees that the key will be of type string. So we just need to worry about
    // deserializing the value here
    LazyMap lazyMap = (LazyMap) LazyFactory.createLazyObject(objectInspector);
    Map map = lazyMap.getMap();
    Map<Object, Object> origMap = (Map) obj;
    ObjectInspector keyObjectInspector = ((MapObjectInspector) objectInspector).getMapKeyObjectInspector();
    ObjectInspector valueObjectInspector = ((MapObjectInspector) objectInspector).getMapValueObjectInspector();
    for (Entry entry : origMap.entrySet()) {
        Object value = entry.getValue();
        map.put(toLazyPrimitiveObject(entry.getKey(), keyObjectInspector), toLazyObject(value, valueObjectInspector));
    }
    return lazyMap;
}
Also used : LazyUnionObjectInspector(org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyUnionObjectInspector) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) LazyListObjectInspector(org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyListObjectInspector) LazySimpleStructObjectInspector(org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) LazyMapObjectInspector(org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) Entry(java.util.Map.Entry) LazyMapObjectInspector(org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) LazyObject(org.apache.hadoop.hive.serde2.lazy.LazyObject) Map(java.util.Map) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap)

Example 33 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project hive by apache.

the class SerdeRandomRowSource method getObjectInspector.

private ObjectInspector getObjectInspector(TypeInfo typeInfo) {
    ObjectInspector objectInspector;
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                final PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) typeInfo;
                objectInspector = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveType);
            }
            break;
        case MAP:
            {
                final MapTypeInfo mapType = (MapTypeInfo) typeInfo;
                final MapObjectInspector mapInspector = ObjectInspectorFactory.getStandardMapObjectInspector(getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
                objectInspector = mapInspector;
            }
            break;
        case LIST:
            {
                final ListTypeInfo listType = (ListTypeInfo) typeInfo;
                final ListObjectInspector listInspector = ObjectInspectorFactory.getStandardListObjectInspector(getObjectInspector(listType.getListElementTypeInfo()));
                objectInspector = listInspector;
            }
            break;
        case STRUCT:
            {
                final StructTypeInfo structType = (StructTypeInfo) typeInfo;
                final List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
                final List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
                for (TypeInfo fieldType : fieldTypes) {
                    fieldInspectors.add(getObjectInspector(fieldType));
                }
                final StructObjectInspector structInspector = ObjectInspectorFactory.getStandardStructObjectInspector(structType.getAllStructFieldNames(), fieldInspectors);
                objectInspector = structInspector;
            }
            break;
        case UNION:
            {
                final UnionTypeInfo unionType = (UnionTypeInfo) typeInfo;
                final List<TypeInfo> fieldTypes = unionType.getAllUnionObjectTypeInfos();
                final List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
                for (TypeInfo fieldType : fieldTypes) {
                    fieldInspectors.add(getObjectInspector(fieldType));
                }
                final UnionObjectInspector unionInspector = ObjectInspectorFactory.getStandardUnionObjectInspector(fieldInspectors);
                objectInspector = unionInspector;
            }
            break;
        default:
            throw new RuntimeException("Unexpected category " + typeInfo.getCategory());
    }
    Preconditions.checkState(objectInspector != null);
    return objectInspector;
}
Also used : WritableIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector) WritableByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) StandardStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector) WritableDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) WritableStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) WritableHiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveCharObjectInspector) WritableHiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveVarcharObjectInspector) WritableBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector) WritableTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector) WritableHiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalDayTimeObjectInspector) WritableShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector) StandardMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector) WritableHiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveIntervalYearMonthObjectInspector) WritableFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector) WritableLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector) StandardUnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector) WritableDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector) WritableHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StandardMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) StandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) List(java.util.List) ArrayList(java.util.ArrayList) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) StandardUnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector) StandardStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo)

Example 34 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project hive by apache.

the class ObjectInspectorConverters method getConvertedOI.

/**
 * Utility function to convert from one object inspector type to another.
 * The output object inspector type should have all fields as settableOI type.
 * The above condition can be violated only if equalsCheck is true and inputOI is
 * equal to outputOI.
 * @param inputOI : input object inspector
 * @param outputOI : output object inspector
 * @param oiSettableProperties : The object inspector to isSettable mapping used to cache
 *                               intermediate results.
 * @param equalsCheck : Do we need to check if the inputOI and outputOI are the same?
 *                      true : If they are the same, we return the object inspector directly.
 *                      false : Do not perform an equality check on inputOI and outputOI
 * @return : The output object inspector containing all settable fields. The return value
 *           can contain non-settable fields only if inputOI equals outputOI and equalsCheck is
 *           true.
 */
public static ObjectInspector getConvertedOI(ObjectInspector inputOI, ObjectInspector outputOI, Map<ObjectInspector, Boolean> oiSettableProperties, boolean equalsCheck) {
    // 2. If the outputOI has all fields settable, return it
    if ((equalsCheck && inputOI.equals(outputOI)) || ObjectInspectorUtils.hasAllFieldsSettable(outputOI, oiSettableProperties) == true) {
        return outputOI;
    }
    // T is settable recursively i.e all the nested fields are also settable.
    switch(outputOI.getCategory()) {
        case PRIMITIVE:
            // Create a writable object inspector for primitive type and return it.
            PrimitiveObjectInspector primOutputOI = (PrimitiveObjectInspector) outputOI;
            return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primOutputOI.getTypeInfo());
        case STRUCT:
            StructObjectInspector structOutputOI = (StructObjectInspector) outputOI;
            // create a standard settable struct object inspector.
            List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
            List<String> structFieldNames = new ArrayList<String>(listFields.size());
            List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(listFields.size());
            for (StructField listField : listFields) {
                structFieldNames.add(listField.getFieldName());
                // We need to make sure that the underlying fields are settable as well.
                // Hence, the recursive call for each field.
                // Note that equalsCheck is false while invoking getConvertedOI() because
                // we need to bypass the initial inputOI.equals(outputOI) check.
                structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(), listField.getFieldObjectInspector(), oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
        case LIST:
            ListObjectInspector listOutputOI = (ListObjectInspector) outputOI;
            // We need to make sure that the list element type is settable.
            return ObjectInspectorFactory.getStandardListObjectInspector(getConvertedOI(listOutputOI.getListElementObjectInspector(), listOutputOI.getListElementObjectInspector(), oiSettableProperties, false));
        case MAP:
            MapObjectInspector mapOutputOI = (MapObjectInspector) outputOI;
            // We need to make sure that the key type and the value types are settable.
            return ObjectInspectorFactory.getStandardMapObjectInspector(getConvertedOI(mapOutputOI.getMapKeyObjectInspector(), mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties, false), getConvertedOI(mapOutputOI.getMapValueObjectInspector(), mapOutputOI.getMapValueObjectInspector(), oiSettableProperties, false));
        case UNION:
            UnionObjectInspector unionOutputOI = (UnionObjectInspector) outputOI;
            // create a standard settable union object inspector
            List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
            List<ObjectInspector> unionFieldObjectInspectors = new ArrayList<ObjectInspector>(unionListFields.size());
            for (ObjectInspector listField : unionListFields) {
                // We need to make sure that all the field associated with the union are settable.
                unionFieldObjectInspectors.add(getConvertedOI(listField, listField, oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardUnionObjectInspector(unionFieldObjectInspectors);
        default:
            // Unsupported in-memory structure.
            throw new RuntimeException("Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet.");
    }
}
Also used : VoidObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector) SettableShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector) SettableTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector) SettableHiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveIntervalDayTimeObjectInspector) SettableBinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector) SettableFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector) SettableHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector) SettableHiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector) SettableDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector) SettableBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBooleanObjectInspector) SettableHiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveCharObjectInspector) SettableLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector) JavaStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector) SettableTimestampLocalTZObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampLocalTZObjectInspector) SettableHiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveIntervalYearMonthObjectInspector) SettableIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector) SettableDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector) WritableStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector) SettableByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector) ArrayList(java.util.ArrayList)

Example 35 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project hive by apache.

the class ObjectInspectorUtils method compare.

/**
 * Compare two objects with their respective ObjectInspectors.
 * if nullValueOpt is MAXVALUE, treat null as maximum value.
 * if nullValueOpt is MINVALUE, treat null as minimum value.
 */
public static int compare(Object o1, ObjectInspector oi1, Object o2, ObjectInspector oi2, MapEqualComparer mapEqualComparer, NullValueOption nullValueOpt) {
    if (oi1.getCategory() != oi2.getCategory()) {
        return oi1.getCategory().compareTo(oi2.getCategory());
    }
    if (o1 == null) {
        return o2 == null ? 0 : nullValueOpt.getCmpReturnValue();
    } else if (o2 == null) {
        return -nullValueOpt.getCmpReturnValue();
    }
    switch(oi1.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1);
                PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2);
                if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) {
                    return poi1.getPrimitiveCategory().compareTo(poi2.getPrimitiveCategory());
                }
                switch(poi1.getPrimitiveCategory()) {
                    case VOID:
                        return 0;
                    case BOOLEAN:
                        {
                            int v1 = ((BooleanObjectInspector) poi1).get(o1) ? 1 : 0;
                            int v2 = ((BooleanObjectInspector) poi2).get(o2) ? 1 : 0;
                            return v1 - v2;
                        }
                    case BYTE:
                        {
                            int v1 = ((ByteObjectInspector) poi1).get(o1);
                            int v2 = ((ByteObjectInspector) poi2).get(o2);
                            return v1 - v2;
                        }
                    case SHORT:
                        {
                            int v1 = ((ShortObjectInspector) poi1).get(o1);
                            int v2 = ((ShortObjectInspector) poi2).get(o2);
                            return v1 - v2;
                        }
                    case INT:
                        {
                            int v1 = ((IntObjectInspector) poi1).get(o1);
                            int v2 = ((IntObjectInspector) poi2).get(o2);
                            return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
                        }
                    case LONG:
                        {
                            long v1 = ((LongObjectInspector) poi1).get(o1);
                            long v2 = ((LongObjectInspector) poi2).get(o2);
                            return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
                        }
                    case FLOAT:
                        {
                            float v1 = ((FloatObjectInspector) poi1).get(o1);
                            float v2 = ((FloatObjectInspector) poi2).get(o2);
                            // The IEEE 754 floating point spec specifies that signed -0.0 and 0.0 should be treated as equal.
                            if (v1 == 0.0f && v2 == 0.0f) {
                                return 0;
                            } else {
                                // Float.compare() treats -0.0 and 0.0 as different
                                return Float.compare(v1, v2);
                            }
                        }
                    case DOUBLE:
                        {
                            double v1 = ((DoubleObjectInspector) poi1).get(o1);
                            double v2 = ((DoubleObjectInspector) poi2).get(o2);
                            // The IEEE 754 floating point spec specifies that signed -0.0 and 0.0 should be treated as equal.
                            if (v1 == 0.0d && v2 == 0.0d) {
                                return 0;
                            } else {
                                // Double.compare() treats -0.0 and 0.0 as different
                                return Double.compare(v1, v2);
                            }
                        }
                    case STRING:
                        {
                            if (poi1.preferWritable() || poi2.preferWritable()) {
                                Text t1 = (Text) poi1.getPrimitiveWritableObject(o1);
                                Text t2 = (Text) poi2.getPrimitiveWritableObject(o2);
                                return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 : t1.compareTo(t2));
                            } else {
                                String s1 = (String) poi1.getPrimitiveJavaObject(o1);
                                String s2 = (String) poi2.getPrimitiveJavaObject(o2);
                                return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1.compareTo(s2));
                            }
                        }
                    case CHAR:
                        {
                            HiveCharWritable t1 = ((HiveCharObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveCharWritable t2 = ((HiveCharObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    case VARCHAR:
                        {
                            HiveVarcharWritable t1 = ((HiveVarcharObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveVarcharWritable t2 = ((HiveVarcharObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    case BINARY:
                        {
                            BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return bw1.compareTo(bw2);
                        }
                    case DATE:
                        {
                            DateWritableV2 d1 = ((DateObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            DateWritableV2 d2 = ((DateObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return d1.compareTo(d2);
                        }
                    case TIMESTAMP:
                        {
                            TimestampWritableV2 t1 = ((TimestampObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            TimestampWritableV2 t2 = ((TimestampObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    case TIMESTAMPLOCALTZ:
                        {
                            TimestampLocalTZWritable tstz1 = ((TimestampLocalTZObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            TimestampLocalTZWritable tstz2 = ((TimestampLocalTZObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return tstz1.compareTo(tstz2);
                        }
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonthWritable i1 = ((HiveIntervalYearMonthObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveIntervalYearMonthWritable i2 = ((HiveIntervalYearMonthObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return i1.compareTo(i2);
                        }
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTimeWritable i1 = ((HiveIntervalDayTimeObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveIntervalDayTimeWritable i2 = ((HiveIntervalDayTimeObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return i1.compareTo(i2);
                        }
                    case DECIMAL:
                        {
                            HiveDecimalWritable t1 = ((HiveDecimalObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveDecimalWritable t2 = ((HiveDecimalObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    default:
                        {
                            throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory());
                        }
                }
            }
        case STRUCT:
            {
                StructObjectInspector soi1 = (StructObjectInspector) oi1;
                StructObjectInspector soi2 = (StructObjectInspector) oi2;
                List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
                List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
                int minimum = Math.min(fields1.size(), fields2.size());
                for (int i = 0; i < minimum; i++) {
                    int r = compare(soi1.getStructFieldData(o1, fields1.get(i)), fields1.get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2, fields2.get(i)), fields2.get(i).getFieldObjectInspector(), mapEqualComparer, nullValueOpt);
                    if (r != 0) {
                        return r;
                    }
                }
                return fields1.size() - fields2.size();
            }
        case LIST:
            {
                ListObjectInspector loi1 = (ListObjectInspector) oi1;
                ListObjectInspector loi2 = (ListObjectInspector) oi2;
                int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2));
                for (int i = 0; i < minimum; i++) {
                    int r = compare(loi1.getListElement(o1, i), loi1.getListElementObjectInspector(), loi2.getListElement(o2, i), loi2.getListElementObjectInspector(), mapEqualComparer, nullValueOpt);
                    if (r != 0) {
                        return r;
                    }
                }
                return loi1.getListLength(o1) - loi2.getListLength(o2);
            }
        case MAP:
            {
                if (mapEqualComparer == null) {
                    throw new RuntimeException("Compare on map type not supported!");
                } else {
                    return mapEqualComparer.compare(o1, (MapObjectInspector) oi1, o2, (MapObjectInspector) oi2);
                }
            }
        case UNION:
            {
                UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
                UnionObjectInspector uoi2 = (UnionObjectInspector) oi2;
                byte tag1 = uoi1.getTag(o1);
                byte tag2 = uoi2.getTag(o2);
                if (tag1 != tag2) {
                    return tag1 - tag2;
                }
                return compare(uoi1.getField(o1), uoi1.getObjectInspectors().get(tag1), uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2), mapEqualComparer, nullValueOpt);
            }
        default:
            throw new RuntimeException("Compare on unknown type: " + oi1.getCategory());
    }
}
Also used : SettableLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableLongObjectInspector) LongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector) DateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector) SettableDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDateObjectInspector) IntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector) SettableIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableIntObjectInspector) SettableBinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableBinaryObjectInspector) BinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector) FloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector) SettableFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableFloatObjectInspector) ByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector) SettableByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableByteObjectInspector) SettableTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampObjectInspector) TimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector) ShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector) SettableShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector) SettableHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveDecimalObjectInspector) HiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector) List(java.util.List) ArrayList(java.util.ArrayList) SettableHiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveCharObjectInspector) HiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) HiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalDayTimeObjectInspector) SettableHiveIntervalDayTimeObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveIntervalDayTimeObjectInspector) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) HiveVarcharWritable(org.apache.hadoop.hive.serde2.io.HiveVarcharWritable) DateWritableV2(org.apache.hadoop.hive.serde2.io.DateWritableV2) SettableTimestampLocalTZObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampLocalTZObjectInspector) TimestampLocalTZObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector) Text(org.apache.hadoop.io.Text) BytesWritable(org.apache.hadoop.io.BytesWritable) HiveIntervalDayTimeWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable) HiveIntervalYearMonthWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable) TimestampWritableV2(org.apache.hadoop.hive.serde2.io.TimestampWritableV2) SettableDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableDoubleObjectInspector) DoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector) HiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector) SettableHiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveVarcharObjectInspector) HiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveIntervalYearMonthObjectInspector) SettableHiveIntervalYearMonthObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableHiveIntervalYearMonthObjectInspector) TimestampLocalTZWritable(org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable)

Aggregations

MapObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector)55 ListObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector)47 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)46 ObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)45 PrimitiveObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector)34 Map (java.util.Map)28 ArrayList (java.util.ArrayList)24 LongObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector)23 BinaryObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector)21 ByteObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector)21 DoubleObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector)21 FloatObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector)21 IntObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector)21 ShortObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector)21 TimestampObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector)21 BooleanObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector)20 StringObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector)20 DateObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector)19 HiveDecimalObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector)19 StructField (org.apache.hadoop.hive.serde2.objectinspector.StructField)17