Examples with PRIMITIVE org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE used on opensource projects

Example 96 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class ObjectInspectorUtils method compareTypes.

/**
 * Compares two types identified by the given object inspectors. This method
 * compares the types as follows:
 * <ol>
 * <li>If the given inspectors do not belong to same category, the result is
 * negative.</li>
 * <li>If the given inspectors are for <code>PRIMITIVE</code> type, the result
 * is the comparison of their type names.</li>
 * <li>If the given inspectors are for <code>LIST</code> type, then the result
 * is recursive call to compare the type of list elements.</li>
 * <li>If the given inspectors are <code>MAP</code> type, then the result is a
 * recursive call to compare the map key and value types.</li>
 * <li>If the given inspectors are <code>STRUCT</code> type, then the result
 * is negative if they do not have the same number of fields. If they do have
 * the same number of fields, the result is a recursive call to compare each
 * of the field types.</li>
 * <li>If none of the above, the result is negative.</li>
 * </ol>
 * @param o1
 * @param o2
 * @return true if the given object inspectors represent the same types.
 */
public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) {
    Category c1 = o1.getCategory();
    Category c2 = o2.getCategory();
    // Return false if categories are not equal
    if (!c1.equals(c2)) {
        return false;
    }
    // If both categories are primitive return the comparison of type names.
    if (c1.equals(Category.PRIMITIVE)) {
        return o1.getTypeName().equals(o2.getTypeName());
    }
    // If lists, recursively compare the list element types
    if (c1.equals(Category.LIST)) {
        ObjectInspector child1 = ((ListObjectInspector) o1).getListElementObjectInspector();
        ObjectInspector child2 = ((ListObjectInspector) o2).getListElementObjectInspector();
        return compareTypes(child1, child2);
    }
    // If maps, recursively compare the key and value types
    if (c1.equals(Category.MAP)) {
        MapObjectInspector mapOI1 = (MapObjectInspector) o1;
        MapObjectInspector mapOI2 = (MapObjectInspector) o2;
        ObjectInspector childKey1 = mapOI1.getMapKeyObjectInspector();
        ObjectInspector childKey2 = mapOI2.getMapKeyObjectInspector();
        if (compareTypes(childKey1, childKey2)) {
            ObjectInspector childVal1 = mapOI1.getMapValueObjectInspector();
            ObjectInspector childVal2 = mapOI2.getMapValueObjectInspector();
            if (compareTypes(childVal1, childVal2)) {
                return true;
            }
        }
        return false;
    }
    // If structs, recursively compare the fields
    if (c1.equals(Category.STRUCT)) {
        StructObjectInspector structOI1 = (StructObjectInspector) o1;
        StructObjectInspector structOI2 = (StructObjectInspector) o2;
        List<? extends StructField> childFieldsList1 = structOI1.getAllStructFieldRefs();
        List<? extends StructField> childFieldsList2 = structOI2.getAllStructFieldRefs();
        if (childFieldsList1 == null && childFieldsList2 == null) {
            return true;
        } else if (childFieldsList1 == null || childFieldsList2 == null) {
            return false;
        } else if (childFieldsList1.size() != childFieldsList2.size()) {
            return false;
        }
        Iterator<? extends StructField> it1 = childFieldsList1.iterator();
        Iterator<? extends StructField> it2 = childFieldsList2.iterator();
        while (it1.hasNext()) {
            StructField field1 = it1.next();
            StructField field2 = it2.next();
            if (!compareTypes(field1.getFieldObjectInspector(), field2.getFieldObjectInspector())) {
                return false;
            }
        }
        return true;
    }
    if (c1.equals(Category.UNION)) {
        UnionObjectInspector uoi1 = (UnionObjectInspector) o1;
        UnionObjectInspector uoi2 = (UnionObjectInspector) o2;
        List<ObjectInspector> ois1 = uoi1.getObjectInspectors();
        List<ObjectInspector> ois2 = uoi2.getObjectInspectors();
        if (ois1 == null && ois2 == null) {
            return true;
        } else if (ois1 == null || ois2 == null) {
            return false;
        } else if (ois1.size() != ois2.size()) {
            return false;
        }
        Iterator<? extends ObjectInspector> it1 = ois1.iterator();
        Iterator<? extends ObjectInspector> it2 = ois2.iterator();
        while (it1.hasNext()) {
            if (!compareTypes(it1.next(), it2.next())) {
                return false;
            }
        }
        return true;
    }
    // Unknown category
    throw new RuntimeException("Unknown category encountered: " + c1);
}

Example 97 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class ObjectInspectorUtils method hashCodeMurmur.

public static int hashCodeMurmur(Object o, ObjectInspector objIns, ByteBuffer byteBuffer) {
    if (o == null) {
        return 0;
    }
    // Reset the bytebuffer
    byteBuffer.clear();
    switch(objIns.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns);
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        return 0;
                    case BOOLEAN:
                        return (((BooleanObjectInspector) poi).get(o) ? 1 : 0);
                    case BYTE:
                        return ((ByteObjectInspector) poi).get(o);
                    case SHORT:
                        {
                            byteBuffer.putShort(((ShortObjectInspector) poi).get(o));
                            return Murmur3.hash32(byteBuffer.array(), 2);
                        }
                    case INT:
                        {
                            byteBuffer.putInt(((IntObjectInspector) poi).get(o));
                            return Murmur3.hash32(byteBuffer.array(), 4);
                        }
                    case LONG:
                        {
                            byteBuffer.putLong(((LongObjectInspector) poi).get(o));
                            return Murmur3.hash32(byteBuffer.array(), 8);
                        }
                    case FLOAT:
                        {
                            byteBuffer.putFloat(Float.floatToIntBits(((FloatObjectInspector) poi).get(o)));
                            return Murmur3.hash32(byteBuffer.array(), 4);
                        }
                    case DOUBLE:
                        {
                            // This hash function returns the same result as Double.hashCode()
                            // while DoubleWritable.hashCode returns a different result.
                            byteBuffer.putDouble(Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o)));
                            return Murmur3.hash32(byteBuffer.array(), 8);
                        }
                    case STRING:
                        {
                            // This hash function returns the same result as String.hashCode() when
                            // all characters are ASCII, while Text.hashCode() always returns a
                            // different result.
                            Text text = ((StringObjectInspector) poi).getPrimitiveWritableObject(o);
                            return Murmur3.hash32(text.getBytes(), text.getLength());
                        }
                    case CHAR:
                        {
                            Text text = ((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).getStrippedValue();
                            return Murmur3.hash32(text.getBytes(), text.getLength());
                        }
                    case VARCHAR:
                        {
                            Text text = ((HiveVarcharObjectInspector) poi).getPrimitiveWritableObject(o).getTextValue();
                            return Murmur3.hash32(text.getBytes(), text.getLength());
                        }
                    case BINARY:
                        return Murmur3.hash32(((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).getBytes());
                    case DATE:
                        byteBuffer.putInt(((DateObjectInspector) poi).getPrimitiveWritableObject(o).getDays());
                        return Murmur3.hash32(byteBuffer.array(), 4);
                    case TIMESTAMP:
                        {
                            TimestampWritableV2 t = ((TimestampObjectInspector) poi).getPrimitiveWritableObject(o);
                            return Murmur3.hash32(t.getBytes());
                        }
                    case TIMESTAMPLOCALTZ:
                        return Murmur3.hash32((((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o)).getBytes());
                    case INTERVAL_YEAR_MONTH:
                        byteBuffer.putInt(((HiveIntervalYearMonthObjectInspector) poi).getPrimitiveWritableObject(o).hashCode());
                        return Murmur3.hash32(byteBuffer.array(), 4);
                    case INTERVAL_DAY_TIME:
                        byteBuffer.putInt(((HiveIntervalDayTimeObjectInspector) poi).getPrimitiveWritableObject(o).hashCode());
                        return Murmur3.hash32(byteBuffer.array(), 4);
                    case DECIMAL:
                        // compatible hash code.
                        return Murmur3.hash32(((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).getInternalStorage());
                    default:
                        {
                            throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                int r = 0;
                ListObjectInspector listOI = (ListObjectInspector) objIns;
                ObjectInspector elemOI = listOI.getListElementObjectInspector();
                for (int ii = 0; ii < listOI.getListLength(o); ++ii) {
                    // r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI);
                    r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI, byteBuffer);
                }
                return r;
            }
        case MAP:
            {
                int r = 0;
                MapObjectInspector mapOI = (MapObjectInspector) objIns;
                ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
                ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
                Map<?, ?> map = mapOI.getMap(o);
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    r += hashCodeMurmur(entry.getKey(), keyOI, byteBuffer) ^ hashCode(entry.getValue(), valueOI);
                }
                return r;
            }
        case STRUCT:
            int r = 0;
            StructObjectInspector structOI = (StructObjectInspector) objIns;
            List<? extends StructField> fields = structOI.getAllStructFieldRefs();
            for (StructField field : fields) {
                r = 31 * r + hashCodeMurmur(structOI.getStructFieldData(o, field), field.getFieldObjectInspector(), byteBuffer);
            }
            return r;
        case UNION:
            UnionObjectInspector uOI = (UnionObjectInspector) objIns;
            byte tag = uOI.getTag(o);
            return hashCodeMurmur(uOI.getField(o), uOI.getObjectInspectors().get(tag), byteBuffer);
        default:
            throw new RuntimeException("Unknown type: " + objIns.getTypeName());
    }
}

Example 98 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class ObjectInspectorUtils method hasAllFieldsSettable.

/**
 * @param oi - Input object inspector
 * @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
 * @return - true if : (1) oi is an instance of settable<DataType>OI.
 *                     (2) All the embedded object inspectors are instances of settable<DataType>OI.
 *           If (1) or (2) is false, return false.
 */
public static boolean hasAllFieldsSettable(ObjectInspector oi, Map<ObjectInspector, Boolean> oiSettableProperties) {
    // If the result is already present in the cache, return it.
    if (!(oiSettableProperties == null) && oiSettableProperties.containsKey(oi)) {
        return oiSettableProperties.get(oi).booleanValue();
    }
    // If the top-level object inspector is non-settable return false
    if (!(isInstanceOfSettableOI(oi))) {
        return setOISettablePropertiesMap(oi, oiSettableProperties, false);
    }
    Boolean returnValue = true;
    switch(oi.getCategory()) {
        case PRIMITIVE:
            break;
        case STRUCT:
            StructObjectInspector structOutputOI = (StructObjectInspector) oi;
            List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
            for (StructField listField : listFields) {
                if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
                    returnValue = false;
                    break;
                }
            }
            break;
        case LIST:
            ListObjectInspector listOutputOI = (ListObjectInspector) oi;
            returnValue = hasAllFieldsSettable(listOutputOI.getListElementObjectInspector(), oiSettableProperties);
            break;
        case MAP:
            MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
            returnValue = hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties) && hasAllFieldsSettable(mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
            break;
        case UNION:
            UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
            List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
            for (ObjectInspector listField : unionListFields) {
                if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
                    returnValue = false;
                    break;
                }
            }
            break;
        default:
            throw new RuntimeException("Hive internal error inside hasAllFieldsSettable : " + oi.getTypeName() + " not supported yet.");
    }
    return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
}

Example 99 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class LazyBinarySerDe2 method createLBSerializer.

/**
 * Generate a LBSerializer for the given ObjectInspector
 * @param oi
 * @return
 */
LBSerializer createLBSerializer(ObjectInspector oi) {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            return createPrimitiveLBSerializer(poi);
        case LIST:
            ListObjectInspector loi = (ListObjectInspector) oi;
            ObjectInspector eoi = loi.getListElementObjectInspector();
            return new LBListSerializer(createLBSerializer(eoi));
        case MAP:
            MapObjectInspector moi = (MapObjectInspector) oi;
            ObjectInspector koi = moi.getMapKeyObjectInspector();
            ObjectInspector voi = moi.getMapValueObjectInspector();
            return new LBMapSerializer(createLBSerializer(koi), createLBSerializer(voi));
        case STRUCT:
            StructObjectInspector soi = (StructObjectInspector) oi;
            List<? extends StructField> fields = soi.getAllStructFieldRefs();
            LBSerializer[] fieldSerializers = new LBSerializer[fields.size()];
            for (int idx = 0; idx < fieldSerializers.length; ++idx) {
                fieldSerializers[idx] = createLBSerializer(fields.get(idx).getFieldObjectInspector());
            }
            return new LBStructSerializer(fieldSerializers);
        case UNION:
            UnionObjectInspector uoi = (UnionObjectInspector) oi;
            List<ObjectInspector> unionFields = uoi.getObjectInspectors();
            LBSerializer[] unionFieldSerializers = new LBSerializer[unionFields.size()];
            for (int idx = 0; idx < unionFieldSerializers.length; ++idx) {
                unionFieldSerializers[idx] = createLBSerializer(unionFields.get(idx));
            }
            return new LBUnionSerializer(unionFieldSerializers);
        default:
            throw new IllegalArgumentException("Unsupported category " + oi.getCategory());
    }
}

Example 100 with PRIMITIVE

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE in project hive by apache.

the class ObjectInspectorFactory method getReflectionObjectInspectorNoCache.

private static ObjectInspector getReflectionObjectInspectorNoCache(Type t, ObjectInspectorOptions options, boolean ensureInited) {
    if (t instanceof GenericArrayType) {
        GenericArrayType at = (GenericArrayType) t;
        return getStandardListObjectInspector(getReflectionObjectInspector(at.getGenericComponentType(), options, ensureInited));
    }
    if (t instanceof ParameterizedType) {
        ParameterizedType pt = (ParameterizedType) t;
        // List?
        if (List.class.isAssignableFrom((Class<?>) pt.getRawType()) || Set.class.isAssignableFrom((Class<?>) pt.getRawType())) {
            return getStandardListObjectInspector(getReflectionObjectInspector(pt.getActualTypeArguments()[0], options, ensureInited));
        }
        // Map?
        if (Map.class.isAssignableFrom((Class<?>) pt.getRawType())) {
            return getStandardMapObjectInspector(getReflectionObjectInspector(pt.getActualTypeArguments()[0], options, ensureInited), getReflectionObjectInspector(pt.getActualTypeArguments()[1], options, ensureInited));
        }
        // Otherwise convert t to RawType so we will fall into the following if
        // block.
        t = pt.getRawType();
    }
    // Must be a class.
    if (!(t instanceof Class)) {
        throw new RuntimeException(ObjectInspectorFactory.class.getName() + " internal error:" + t);
    }
    Class<?> c = (Class<?>) t;
    // Java Primitive Type?
    if (PrimitiveObjectInspectorUtils.isPrimitiveJavaType(c)) {
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaType(c).primitiveCategory);
    }
    // Java Primitive Class?
    if (PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(c)) {
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveJavaClass(c).primitiveCategory);
    }
    // Primitive Writable class?
    if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(c)) {
        return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(PrimitiveObjectInspectorUtils.getTypeEntryFromPrimitiveWritableClass(c).primitiveCategory);
    }
    // Enum class?
    if (Enum.class.isAssignableFrom(c)) {
        return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
    }
    // Must be struct because List and Map need to be ParameterizedType
    assert (!List.class.isAssignableFrom(c));
    assert (!Map.class.isAssignableFrom(c));
    // Create StructObjectInspector
    ReflectionStructObjectInspector oi;
    switch(options) {
        case JAVA:
            oi = new ReflectionStructObjectInspector();
            break;
        case THRIFT:
            oi = TUnion.class.isAssignableFrom(c) ? new ThriftUnionObjectInspector() : new ThriftStructObjectInspector();
            break;
        case PROTOCOL_BUFFERS:
            oi = new ProtocolBuffersStructObjectInspector();
            break;
        default:
            throw new RuntimeException(ObjectInspectorFactory.class.getName() + ": internal error.");
    }
    // put it into the cache BEFORE it is initialized to make sure we can catch
    // recursive types.
    ReflectionStructObjectInspector prev = (ReflectionStructObjectInspector) objectInspectorCache.asMap().putIfAbsent(t, oi);
    if (prev != null) {
        oi = prev;
    } else {
        try {
            oi.init(t, c, options);
        } finally {
            if (!oi.inited) {
                // Failed to init, remove it from cache
                objectInspectorCache.asMap().remove(t, oi);
            }
        }
    }
    return oi;
}