Search in sources :

Example 1 with LazyBinaryMapObjectInspector

use of org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector in project hive by apache.

the class TestLazyBinarySerDe method testLazyBinaryMap.

void testLazyBinaryMap(Random r) throws Throwable {
    StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClassBigger.class, ObjectInspectorOptions.JAVA);
    String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
    AbstractSerDe serde = getSerDe(fieldNames, fieldTypes);
    ObjectInspector serdeOI = serde.getObjectInspector();
    StructObjectInspector soi1 = (StructObjectInspector) serdeOI;
    List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
    LazyBinaryMapObjectInspector lazympoi = (LazyBinaryMapObjectInspector) fields1.get(MyTestClassBigger.mapPos).getFieldObjectInspector();
    ObjectInspector lazympkeyoi = lazympoi.getMapKeyObjectInspector();
    ObjectInspector lazympvalueoi = lazympoi.getMapValueObjectInspector();
    StructObjectInspector soi2 = rowOI;
    List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
    MapObjectInspector inputmpoi = (MapObjectInspector) fields2.get(MyTestClassBigger.mapPos).getFieldObjectInspector();
    ObjectInspector inputmpkeyoi = inputmpoi.getMapKeyObjectInspector();
    ObjectInspector inputmpvalueoi = inputmpoi.getMapValueObjectInspector();
    int num = 100;
    for (int testi = 0; testi < num; testi++) {
        Map<String, List<MyTestInnerStruct>> mp = new LinkedHashMap<String, List<MyTestInnerStruct>>();
        int randFields = r.nextInt(10);
        for (int i = 0; i < randFields; i++) {
            String key = MyTestPrimitiveClass.getRandString(r);
            int randField = r.nextInt(10);
            List<MyTestInnerStruct> value = randField > 4 ? null : getRandStructArray(r);
            mp.put(key, value);
        }
        MyTestClassBigger t = new MyTestClassBigger();
        t.myMap = mp;
        BytesWritable bw = (BytesWritable) serde.serialize(t, rowOI);
        Object output = serde.deserialize(bw);
        Object lazyobj = soi1.getStructFieldData(output, fields1.get(MyTestClassBigger.mapPos));
        Map<?, ?> outputmp = lazympoi.getMap(lazyobj);
        if (outputmp.size() != mp.size()) {
            throw new RuntimeException("Map size changed from " + mp.size() + " to " + outputmp.size() + " after serialization!");
        }
        for (Map.Entry<?, ?> entryinput : mp.entrySet()) {
            boolean bEqual = false;
            for (Map.Entry<?, ?> entryoutput : outputmp.entrySet()) {
                // find the same key
                if (0 == ObjectInspectorUtils.compare(entryoutput.getKey(), lazympkeyoi, entryinput.getKey(), inputmpkeyoi)) {
                    if (0 != ObjectInspectorUtils.compare(entryoutput.getValue(), lazympvalueoi, entryinput.getValue(), inputmpvalueoi)) {
                        assertEquals(entryoutput.getValue(), entryinput.getValue());
                    } else {
                        bEqual = true;
                    }
                    break;
                }
            }
            if (!bEqual) {
                throw new RuntimeException("Could not find matched key in deserialized map : " + entryinput.getKey());
            }
        }
    }
}
Also used : AbstractPrimitiveLazyObjectInspector(org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector) LazyBinaryMapObjectInspector(org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector) WritableBinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) JavaBinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBinaryObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) MyTestInnerStruct(org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct) LazyBinaryMapObjectInspector(org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector) BytesWritable(org.apache.hadoop.io.BytesWritable) AbstractSerDe(org.apache.hadoop.hive.serde2.AbstractSerDe) LinkedHashMap(java.util.LinkedHashMap) LazyBinaryMapObjectInspector(org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) ArrayList(java.util.ArrayList) List(java.util.List) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)

Aggregations

ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 LinkedHashMap (java.util.LinkedHashMap)1 List (java.util.List)1 Map (java.util.Map)1 AbstractSerDe (org.apache.hadoop.hive.serde2.AbstractSerDe)1 MyTestInnerStruct (org.apache.hadoop.hive.serde2.binarysortable.MyTestInnerStruct)1 AbstractPrimitiveLazyObjectInspector (org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector)1 LazyBinaryMapObjectInspector (org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector)1 MapObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector)1 ObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)1 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)1 JavaBinaryObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBinaryObjectInspector)1 WritableBinaryObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector)1 BytesWritable (org.apache.hadoop.io.BytesWritable)1