Examples with UnionObjectInspector org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector used on opensource projects

Example 11 with UnionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector in project hive by apache.

the class LazyBinarySerDe2 method createLBSerializer.

/**
 * Generate a LBSerializer for the given ObjectInspector
 * @param oi
 * @return
 */
LBSerializer createLBSerializer(ObjectInspector oi) {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            return createPrimitiveLBSerializer(poi);
        case LIST:
            ListObjectInspector loi = (ListObjectInspector) oi;
            ObjectInspector eoi = loi.getListElementObjectInspector();
            return new LBListSerializer(createLBSerializer(eoi));
        case MAP:
            MapObjectInspector moi = (MapObjectInspector) oi;
            ObjectInspector koi = moi.getMapKeyObjectInspector();
            ObjectInspector voi = moi.getMapValueObjectInspector();
            return new LBMapSerializer(createLBSerializer(koi), createLBSerializer(voi));
        case STRUCT:
            StructObjectInspector soi = (StructObjectInspector) oi;
            List<? extends StructField> fields = soi.getAllStructFieldRefs();
            LBSerializer[] fieldSerializers = new LBSerializer[fields.size()];
            for (int idx = 0; idx < fieldSerializers.length; ++idx) {
                fieldSerializers[idx] = createLBSerializer(fields.get(idx).getFieldObjectInspector());
            }
            return new LBStructSerializer(fieldSerializers);
        case UNION:
            UnionObjectInspector uoi = (UnionObjectInspector) oi;
            List<ObjectInspector> unionFields = uoi.getObjectInspectors();
            LBSerializer[] unionFieldSerializers = new LBSerializer[unionFields.size()];
            for (int idx = 0; idx < unionFieldSerializers.length; ++idx) {
                unionFieldSerializers[idx] = createLBSerializer(unionFields.get(idx));
            }
            return new LBUnionSerializer(unionFieldSerializers);
        default:
            throw new IllegalArgumentException("Unsupported category " + oi.getCategory());
    }
}

Example 12 with UnionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector in project hive by apache.

the class ObjectInspectorConverters method getConvertedOI.

/**
 * Utility function to convert from one object inspector type to another.
 * The output object inspector type should have all fields as settableOI type.
 * The above condition can be violated only if equalsCheck is true and inputOI is
 * equal to outputOI.
 * @param inputOI : input object inspector
 * @param outputOI : output object inspector
 * @param oiSettableProperties : The object inspector to isSettable mapping used to cache
 *                               intermediate results.
 * @param equalsCheck : Do we need to check if the inputOI and outputOI are the same?
 *                      true : If they are the same, we return the object inspector directly.
 *                      false : Do not perform an equality check on inputOI and outputOI
 * @return : The output object inspector containing all settable fields. The return value
 *           can contain non-settable fields only if inputOI equals outputOI and equalsCheck is
 *           true.
 */
public static ObjectInspector getConvertedOI(ObjectInspector inputOI, ObjectInspector outputOI, Map<ObjectInspector, Boolean> oiSettableProperties, boolean equalsCheck) {
    // 2. If the outputOI has all fields settable, return it
    if ((equalsCheck && inputOI.equals(outputOI)) || ObjectInspectorUtils.hasAllFieldsSettable(outputOI, oiSettableProperties) == true) {
        return outputOI;
    }
    // T is settable recursively i.e all the nested fields are also settable.
    switch(outputOI.getCategory()) {
        case PRIMITIVE:
            // Create a writable object inspector for primitive type and return it.
            PrimitiveObjectInspector primOutputOI = (PrimitiveObjectInspector) outputOI;
            return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primOutputOI.getTypeInfo());
        case STRUCT:
            StructObjectInspector structOutputOI = (StructObjectInspector) outputOI;
            // create a standard settable struct object inspector.
            List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
            List<String> structFieldNames = new ArrayList<String>(listFields.size());
            List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(listFields.size());
            for (StructField listField : listFields) {
                structFieldNames.add(listField.getFieldName());
                // We need to make sure that the underlying fields are settable as well.
                // Hence, the recursive call for each field.
                // Note that equalsCheck is false while invoking getConvertedOI() because
                // we need to bypass the initial inputOI.equals(outputOI) check.
                structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(), listField.getFieldObjectInspector(), oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
        case LIST:
            ListObjectInspector listOutputOI = (ListObjectInspector) outputOI;
            // We need to make sure that the list element type is settable.
            return ObjectInspectorFactory.getStandardListObjectInspector(getConvertedOI(listOutputOI.getListElementObjectInspector(), listOutputOI.getListElementObjectInspector(), oiSettableProperties, false));
        case MAP:
            MapObjectInspector mapOutputOI = (MapObjectInspector) outputOI;
            // We need to make sure that the key type and the value types are settable.
            return ObjectInspectorFactory.getStandardMapObjectInspector(getConvertedOI(mapOutputOI.getMapKeyObjectInspector(), mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties, false), getConvertedOI(mapOutputOI.getMapValueObjectInspector(), mapOutputOI.getMapValueObjectInspector(), oiSettableProperties, false));
        case UNION:
            UnionObjectInspector unionOutputOI = (UnionObjectInspector) outputOI;
            // create a standard settable union object inspector
            List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
            List<ObjectInspector> unionFieldObjectInspectors = new ArrayList<ObjectInspector>(unionListFields.size());
            for (ObjectInspector listField : unionListFields) {
                // We need to make sure that all the field associated with the union are settable.
                unionFieldObjectInspectors.add(getConvertedOI(listField, listField, oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardUnionObjectInspector(unionFieldObjectInspectors);
        default:
            // Unsupported in-memory structure.
            throw new RuntimeException("Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet.");
    }
}

Example 13 with UnionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector in project hive by apache.

the class ObjectInspectorUtils method hasAllFieldsSettable.

/**
 * @param oi - Input object inspector
 * @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
 * @return - true if : (1) oi is an instance of settable<DataType>OI.
 *                     (2) All the embedded object inspectors are instances of settable<DataType>OI.
 *           If (1) or (2) is false, return false.
 */
public static boolean hasAllFieldsSettable(ObjectInspector oi, Map<ObjectInspector, Boolean> oiSettableProperties) {
    // If the result is already present in the cache, return it.
    if (!(oiSettableProperties == null) && oiSettableProperties.containsKey(oi)) {
        return oiSettableProperties.get(oi).booleanValue();
    }
    // If the top-level object inspector is non-settable return false
    if (!(isInstanceOfSettableOI(oi))) {
        return setOISettablePropertiesMap(oi, oiSettableProperties, false);
    }
    Boolean returnValue = true;
    switch(oi.getCategory()) {
        case PRIMITIVE:
            break;
        case STRUCT:
            StructObjectInspector structOutputOI = (StructObjectInspector) oi;
            List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
            for (StructField listField : listFields) {
                if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
                    returnValue = false;
                    break;
                }
            }
            break;
        case LIST:
            ListObjectInspector listOutputOI = (ListObjectInspector) oi;
            returnValue = hasAllFieldsSettable(listOutputOI.getListElementObjectInspector(), oiSettableProperties);
            break;
        case MAP:
            MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
            returnValue = hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties) && hasAllFieldsSettable(mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
            break;
        case UNION:
            UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
            List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
            for (ObjectInspector listField : unionListFields) {
                if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
                    returnValue = false;
                    break;
                }
            }
            break;
        default:
            throw new RuntimeException("Hive internal error inside hasAllFieldsSettable : " + oi.getTypeName() + " not supported yet.");
    }
    return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
}

Example 14 with UnionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector in project hive by apache.

the class ObjectInspectorUtils method compareTypes.

/**
 * Compares two types identified by the given object inspectors. This method
 * compares the types as follows:
 * <ol>
 * <li>If the given inspectors do not belong to same category, the result is
 * negative.</li>
 * <li>If the given inspectors are for <code>PRIMITIVE</code> type, the result
 * is the comparison of their type names.</li>
 * <li>If the given inspectors are for <code>LIST</code> type, then the result
 * is recursive call to compare the type of list elements.</li>
 * <li>If the given inspectors are <code>MAP</code> type, then the result is a
 * recursive call to compare the map key and value types.</li>
 * <li>If the given inspectors are <code>STRUCT</code> type, then the result
 * is negative if they do not have the same number of fields. If they do have
 * the same number of fields, the result is a recursive call to compare each
 * of the field types.</li>
 * <li>If none of the above, the result is negative.</li>
 * </ol>
 * @param o1
 * @param o2
 * @return true if the given object inspectors represent the same types.
 */
public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) {
    Category c1 = o1.getCategory();
    Category c2 = o2.getCategory();
    // Return false if categories are not equal
    if (!c1.equals(c2)) {
        return false;
    }
    // If both categories are primitive return the comparison of type names.
    if (c1.equals(Category.PRIMITIVE)) {
        return o1.getTypeName().equals(o2.getTypeName());
    }
    // If lists, recursively compare the list element types
    if (c1.equals(Category.LIST)) {
        ObjectInspector child1 = ((ListObjectInspector) o1).getListElementObjectInspector();
        ObjectInspector child2 = ((ListObjectInspector) o2).getListElementObjectInspector();
        return compareTypes(child1, child2);
    }
    // If maps, recursively compare the key and value types
    if (c1.equals(Category.MAP)) {
        MapObjectInspector mapOI1 = (MapObjectInspector) o1;
        MapObjectInspector mapOI2 = (MapObjectInspector) o2;
        ObjectInspector childKey1 = mapOI1.getMapKeyObjectInspector();
        ObjectInspector childKey2 = mapOI2.getMapKeyObjectInspector();
        if (compareTypes(childKey1, childKey2)) {
            ObjectInspector childVal1 = mapOI1.getMapValueObjectInspector();
            ObjectInspector childVal2 = mapOI2.getMapValueObjectInspector();
            if (compareTypes(childVal1, childVal2)) {
                return true;
            }
        }
        return false;
    }
    // If structs, recursively compare the fields
    if (c1.equals(Category.STRUCT)) {
        StructObjectInspector structOI1 = (StructObjectInspector) o1;
        StructObjectInspector structOI2 = (StructObjectInspector) o2;
        List<? extends StructField> childFieldsList1 = structOI1.getAllStructFieldRefs();
        List<? extends StructField> childFieldsList2 = structOI2.getAllStructFieldRefs();
        if (childFieldsList1 == null && childFieldsList2 == null) {
            return true;
        } else if (childFieldsList1 == null || childFieldsList2 == null) {
            return false;
        } else if (childFieldsList1.size() != childFieldsList2.size()) {
            return false;
        }
        Iterator<? extends StructField> it1 = childFieldsList1.iterator();
        Iterator<? extends StructField> it2 = childFieldsList2.iterator();
        while (it1.hasNext()) {
            StructField field1 = it1.next();
            StructField field2 = it2.next();
            if (!compareTypes(field1.getFieldObjectInspector(), field2.getFieldObjectInspector())) {
                return false;
            }
        }
        return true;
    }
    if (c1.equals(Category.UNION)) {
        UnionObjectInspector uoi1 = (UnionObjectInspector) o1;
        UnionObjectInspector uoi2 = (UnionObjectInspector) o2;
        List<ObjectInspector> ois1 = uoi1.getObjectInspectors();
        List<ObjectInspector> ois2 = uoi2.getObjectInspectors();
        if (ois1 == null && ois2 == null) {
            return true;
        } else if (ois1 == null || ois2 == null) {
            return false;
        } else if (ois1.size() != ois2.size()) {
            return false;
        }
        Iterator<? extends ObjectInspector> it1 = ois1.iterator();
        Iterator<? extends ObjectInspector> it2 = ois2.iterator();
        while (it1.hasNext()) {
            if (!compareTypes(it1.next(), it2.next())) {
                return false;
            }
        }
        return true;
    }
    // Unknown category
    throw new RuntimeException("Unknown category encountered: " + c1);
}

Example 15 with UnionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector in project hive by apache.

the class ObjectInspectorUtils method compare.

/**
 * Compare two objects with their respective ObjectInspectors.
 * if nullValueOpt is MAXVALUE, treat null as maximum value.
 * if nullValueOpt is MINVALUE, treat null as minimum value.
 */
public static int compare(Object o1, ObjectInspector oi1, Object o2, ObjectInspector oi2, MapEqualComparer mapEqualComparer, NullValueOption nullValueOpt) {
    if (oi1.getCategory() != oi2.getCategory()) {
        return oi1.getCategory().compareTo(oi2.getCategory());
    }
    int nullCmpRtn = -1;
    switch(nullValueOpt) {
        case MAXVALUE:
            nullCmpRtn = 1;
            break;
        case MINVALUE:
            nullCmpRtn = -1;
            break;
    }
    if (o1 == null) {
        return o2 == null ? 0 : nullCmpRtn;
    } else if (o2 == null) {
        return -nullCmpRtn;
    }
    switch(oi1.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1);
                PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2);
                if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) {
                    return poi1.getPrimitiveCategory().compareTo(poi2.getPrimitiveCategory());
                }
                switch(poi1.getPrimitiveCategory()) {
                    case VOID:
                        return 0;
                    case BOOLEAN:
                        {
                            int v1 = ((BooleanObjectInspector) poi1).get(o1) ? 1 : 0;
                            int v2 = ((BooleanObjectInspector) poi2).get(o2) ? 1 : 0;
                            return v1 - v2;
                        }
                    case BYTE:
                        {
                            int v1 = ((ByteObjectInspector) poi1).get(o1);
                            int v2 = ((ByteObjectInspector) poi2).get(o2);
                            return v1 - v2;
                        }
                    case SHORT:
                        {
                            int v1 = ((ShortObjectInspector) poi1).get(o1);
                            int v2 = ((ShortObjectInspector) poi2).get(o2);
                            return v1 - v2;
                        }
                    case INT:
                        {
                            int v1 = ((IntObjectInspector) poi1).get(o1);
                            int v2 = ((IntObjectInspector) poi2).get(o2);
                            return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
                        }
                    case LONG:
                        {
                            long v1 = ((LongObjectInspector) poi1).get(o1);
                            long v2 = ((LongObjectInspector) poi2).get(o2);
                            return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
                        }
                    case FLOAT:
                        {
                            float v1 = ((FloatObjectInspector) poi1).get(o1);
                            float v2 = ((FloatObjectInspector) poi2).get(o2);
                            // The IEEE 754 floating point spec specifies that signed -0.0 and 0.0 should be treated as equal.
                            if (v1 == 0.0f && v2 == 0.0f) {
                                return 0;
                            } else {
                                // Float.compare() treats -0.0 and 0.0 as different
                                return Float.compare(v1, v2);
                            }
                        }
                    case DOUBLE:
                        {
                            double v1 = ((DoubleObjectInspector) poi1).get(o1);
                            double v2 = ((DoubleObjectInspector) poi2).get(o2);
                            // The IEEE 754 floating point spec specifies that signed -0.0 and 0.0 should be treated as equal.
                            if (v1 == 0.0d && v2 == 0.0d) {
                                return 0;
                            } else {
                                // Double.compare() treats -0.0 and 0.0 as different
                                return Double.compare(v1, v2);
                            }
                        }
                    case STRING:
                        {
                            if (poi1.preferWritable() || poi2.preferWritable()) {
                                Text t1 = (Text) poi1.getPrimitiveWritableObject(o1);
                                Text t2 = (Text) poi2.getPrimitiveWritableObject(o2);
                                return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 : t1.compareTo(t2));
                            } else {
                                String s1 = (String) poi1.getPrimitiveJavaObject(o1);
                                String s2 = (String) poi2.getPrimitiveJavaObject(o2);
                                return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1.compareTo(s2));
                            }
                        }
                    case CHAR:
                        {
                            HiveCharWritable t1 = ((HiveCharObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveCharWritable t2 = ((HiveCharObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    case VARCHAR:
                        {
                            HiveVarcharWritable t1 = ((HiveVarcharObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveVarcharWritable t2 = ((HiveVarcharObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    case BINARY:
                        {
                            BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return bw1.compareTo(bw2);
                        }
                    case DATE:
                        {
                            DateWritable d1 = ((DateObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            DateWritable d2 = ((DateObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return d1.compareTo(d2);
                        }
                    case TIMESTAMP:
                        {
                            TimestampWritable t1 = ((TimestampObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            TimestampWritable t2 = ((TimestampObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    case TIMESTAMPLOCALTZ:
                        {
                            TimestampLocalTZWritable tstz1 = ((TimestampLocalTZObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            TimestampLocalTZWritable tstz2 = ((TimestampLocalTZObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return tstz1.compareTo(tstz2);
                        }
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonthWritable i1 = ((HiveIntervalYearMonthObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveIntervalYearMonthWritable i2 = ((HiveIntervalYearMonthObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return i1.compareTo(i2);
                        }
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTimeWritable i1 = ((HiveIntervalDayTimeObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveIntervalDayTimeWritable i2 = ((HiveIntervalDayTimeObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return i1.compareTo(i2);
                        }
                    case DECIMAL:
                        {
                            HiveDecimalWritable t1 = ((HiveDecimalObjectInspector) poi1).getPrimitiveWritableObject(o1);
                            HiveDecimalWritable t2 = ((HiveDecimalObjectInspector) poi2).getPrimitiveWritableObject(o2);
                            return t1.compareTo(t2);
                        }
                    default:
                        {
                            throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory());
                        }
                }
            }
        case STRUCT:
            {
                StructObjectInspector soi1 = (StructObjectInspector) oi1;
                StructObjectInspector soi2 = (StructObjectInspector) oi2;
                List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
                List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
                int minimum = Math.min(fields1.size(), fields2.size());
                for (int i = 0; i < minimum; i++) {
                    int r = compare(soi1.getStructFieldData(o1, fields1.get(i)), fields1.get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2, fields2.get(i)), fields2.get(i).getFieldObjectInspector(), mapEqualComparer, nullValueOpt);
                    if (r != 0) {
                        return r;
                    }
                }
                return fields1.size() - fields2.size();
            }
        case LIST:
            {
                ListObjectInspector loi1 = (ListObjectInspector) oi1;
                ListObjectInspector loi2 = (ListObjectInspector) oi2;
                int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2));
                for (int i = 0; i < minimum; i++) {
                    int r = compare(loi1.getListElement(o1, i), loi1.getListElementObjectInspector(), loi2.getListElement(o2, i), loi2.getListElementObjectInspector(), mapEqualComparer, nullValueOpt);
                    if (r != 0) {
                        return r;
                    }
                }
                return loi1.getListLength(o1) - loi2.getListLength(o2);
            }
        case MAP:
            {
                if (mapEqualComparer == null) {
                    throw new RuntimeException("Compare on map type not supported!");
                } else {
                    return mapEqualComparer.compare(o1, (MapObjectInspector) oi1, o2, (MapObjectInspector) oi2);
                }
            }
        case UNION:
            {
                UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
                UnionObjectInspector uoi2 = (UnionObjectInspector) oi2;
                byte tag1 = uoi1.getTag(o1);
                byte tag2 = uoi2.getTag(o2);
                if (tag1 != tag2) {
                    return tag1 - tag2;
                }
                return compare(uoi1.getField(o1), uoi1.getObjectInspectors().get(tag1), uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2), mapEqualComparer, nullValueOpt);
            }
        default:
            throw new RuntimeException("Compare on unknown type: " + oi1.getCategory());
    }
}