Examples with ObjectInspectorFactory.getStandardMapObjectInspector org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector used on opensource projects

Example 1 with ObjectInspectorFactory.getStandardMapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector in project hive by apache.

the class TestGenericUDFSortArray method testSortMap.

@Test
public void testSortMap() throws HiveException {
    ObjectInspector[] inputOIs = { ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.writableStringObjectInspector, PrimitiveObjectInspectorFactory.writableIntObjectInspector)) };
    udf.initialize(inputOIs);
    Map<Text, IntWritable> m1 = new HashMap<Text, IntWritable>();
    m1.put(new Text("a"), new IntWritable(4));
    m1.put(new Text("b"), new IntWritable(3));
    m1.put(new Text("c"), new IntWritable(1));
    m1.put(new Text("d"), new IntWritable(2));
    Map<Text, IntWritable> m2 = new HashMap<Text, IntWritable>();
    m2.put(new Text("d"), new IntWritable(4));
    m2.put(new Text("b"), new IntWritable(3));
    m2.put(new Text("a"), new IntWritable(1));
    m2.put(new Text("c"), new IntWritable(2));
    Map<Text, IntWritable> m3 = new HashMap<Text, IntWritable>();
    m3.put(new Text("d"), new IntWritable(4));
    m3.put(new Text("b"), new IntWritable(3));
    m3.put(new Text("a"), new IntWritable(1));
    runAndVerify(asList((Object) m1, m2, m3), asList((Object) m3, m2, m1));
}

Example 2 with ObjectInspectorFactory.getStandardMapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector in project hive by apache.

the class ObjectInspectorConverters method getConvertedOI.

/**
   * Utility function to convert from one object inspector type to another.
   * The output object inspector type should have all fields as settableOI type.
   * The above condition can be violated only if equalsCheck is true and inputOI is
   * equal to outputOI.
   * @param inputOI : input object inspector
   * @param outputOI : output object inspector
   * @param oiSettableProperties : The object inspector to isSettable mapping used to cache
   *                               intermediate results.
   * @param equalsCheck : Do we need to check if the inputOI and outputOI are the same?
   *                      true : If they are the same, we return the object inspector directly.
   *                      false : Do not perform an equality check on inputOI and outputOI
   * @return : The output object inspector containing all settable fields. The return value
   *           can contain non-settable fields only if inputOI equals outputOI and equalsCheck is
   *           true.
   */
public static ObjectInspector getConvertedOI(ObjectInspector inputOI, ObjectInspector outputOI, Map<ObjectInspector, Boolean> oiSettableProperties, boolean equalsCheck) {
    // 2. If the outputOI has all fields settable, return it
    if ((equalsCheck && inputOI.equals(outputOI)) || ObjectInspectorUtils.hasAllFieldsSettable(outputOI, oiSettableProperties) == true) {
        return outputOI;
    }
    // T is settable recursively i.e all the nested fields are also settable.
    switch(outputOI.getCategory()) {
        case PRIMITIVE:
            // Create a writable object inspector for primitive type and return it.
            PrimitiveObjectInspector primOutputOI = (PrimitiveObjectInspector) outputOI;
            return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primOutputOI.getTypeInfo());
        case STRUCT:
            StructObjectInspector structOutputOI = (StructObjectInspector) outputOI;
            // create a standard settable struct object inspector.
            List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
            List<String> structFieldNames = new ArrayList<String>(listFields.size());
            List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(listFields.size());
            for (StructField listField : listFields) {
                structFieldNames.add(listField.getFieldName());
                // We need to make sure that the underlying fields are settable as well.
                // Hence, the recursive call for each field.
                // Note that equalsCheck is false while invoking getConvertedOI() because
                // we need to bypass the initial inputOI.equals(outputOI) check.
                structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(), listField.getFieldObjectInspector(), oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
        case LIST:
            ListObjectInspector listOutputOI = (ListObjectInspector) outputOI;
            // We need to make sure that the list element type is settable.
            return ObjectInspectorFactory.getStandardListObjectInspector(getConvertedOI(listOutputOI.getListElementObjectInspector(), listOutputOI.getListElementObjectInspector(), oiSettableProperties, false));
        case MAP:
            MapObjectInspector mapOutputOI = (MapObjectInspector) outputOI;
            // We need to make sure that the key type and the value types are settable.
            return ObjectInspectorFactory.getStandardMapObjectInspector(getConvertedOI(mapOutputOI.getMapKeyObjectInspector(), mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties, false), getConvertedOI(mapOutputOI.getMapValueObjectInspector(), mapOutputOI.getMapValueObjectInspector(), oiSettableProperties, false));
        case UNION:
            UnionObjectInspector unionOutputOI = (UnionObjectInspector) outputOI;
            // create a standard settable union object inspector
            List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
            List<ObjectInspector> unionFieldObjectInspectors = new ArrayList<ObjectInspector>(unionListFields.size());
            for (ObjectInspector listField : unionListFields) {
                // We need to make sure that all the field associated with the union are settable.
                unionFieldObjectInspectors.add(getConvertedOI(listField, listField, oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardUnionObjectInspector(unionFieldObjectInspectors);
        default:
            // Unsupported in-memory structure.
            throw new RuntimeException("Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet.");
    }
}

Example 3 with ObjectInspectorFactory.getStandardMapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector in project hive by apache.

the class TestObjectInspectorUtils method testObjectInspectorUtils.

public void testObjectInspectorUtils() throws Throwable {
    try {
        ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        // metadata
        assertEquals(Category.STRUCT, oi1.getCategory());
        // standard ObjectInspector
        StructObjectInspector soi = (StructObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(oi1);
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        assertEquals(10, fields.size());
        assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
        // null
        for (int i = 0; i < fields.size(); i++) {
            assertNull(soi.getStructFieldData(null, fields.get(i)));
        }
        // real object
        Complex cc = new Complex();
        cc.setAint(1);
        cc.setAString("test");
        List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
        cc.setLint(c2);
        List<String> c3 = Arrays.asList(new String[] { "one", "two" });
        cc.setLString(c3);
        List<IntString> c4 = new ArrayList<IntString>();
        cc.setLintString(c4);
        cc.setMStringString(null);
        // standard object
        Object c = ObjectInspectorUtils.copyToStandardObject(cc, oi1);
        assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
        assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
        assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
        assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
        assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
        assertNull(soi.getStructFieldData(c, fields.get(5)));
        ArrayList<Object> cfields = new ArrayList<Object>();
        for (int i = 0; i < 10; i++) {
            cfields.add(soi.getStructFieldData(c, fields.get(i)));
        }
        assertEquals(cfields, soi.getStructFieldsDataAsList(c));
        // sub fields
        assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
        assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
        assertEquals(ObjectInspectorUtils.getStandardObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT))), fields.get(4).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(5).getFieldObjectInspector());
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 4 with ObjectInspectorFactory.getStandardMapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector in project hive by apache.

the class TestThriftObjectInspectors method testThriftObjectInspectors.

public void testThriftObjectInspectors() throws Throwable {
    try {
        ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        assertEquals(oi1, oi2);
        // metadata
        assertEquals(Category.STRUCT, oi1.getCategory());
        StructObjectInspector soi = (StructObjectInspector) oi1;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        assertEquals(10, fields.size());
        assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
        // null
        for (int i = 0; i < fields.size(); i++) {
            assertNull(soi.getStructFieldData(null, fields.get(i)));
        }
        ObjectInspector oi = ObjectInspectorFactory.getReflectionObjectInspector(PropValueUnion.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        assertNotNull(oi.toString());
        // real object
        Complex c = new Complex();
        c.setAint(1);
        c.setAString("test");
        List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
        c.setLint(c2);
        List<String> c3 = Arrays.asList(new String[] { "one", "two" });
        c.setLString(c3);
        List<IntString> c4 = new ArrayList<IntString>();
        c.setLintString(c4);
        c.setMStringString(null);
        c.setAttributes(null);
        c.setUnionField1(null);
        c.setUnionField2(null);
        c.setUnionField3(null);
        assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
        assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
        assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
        assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
        assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
        assertNull(soi.getStructFieldData(c, fields.get(5)));
        assertNull(soi.getStructFieldData(c, fields.get(6)));
        assertNull(soi.getStructFieldData(c, fields.get(7)));
        assertNull(soi.getStructFieldData(c, fields.get(8)));
        assertNull(soi.getStructFieldData(c, fields.get(9)));
        ArrayList<Object> cfields = new ArrayList<Object>();
        for (int i = 0; i < 10; i++) {
            cfields.add(soi.getStructFieldData(c, fields.get(i)));
        }
        assertEquals(cfields, soi.getStructFieldsDataAsList(c));
        // sub fields
        assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
        assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)), fields.get(4).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(5).getFieldObjectInspector());
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 5 with ObjectInspectorFactory.getStandardMapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector in project hive by apache.

the class GenericUDFMap method initialize.

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    if (arguments.length % 2 != 0) {
        throw new UDFArgumentLengthException("Arguments must be in key/value pairs");
    }
    GenericUDFUtils.ReturnObjectInspectorResolver keyOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
    GenericUDFUtils.ReturnObjectInspectorResolver valueOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
    for (int i = 0; i < arguments.length; i++) {
        if (i % 2 == 0) {
            // Keys
            if (!(arguments[i] instanceof PrimitiveObjectInspector)) {
                throw new UDFArgumentTypeException(1, "Primitive Type is expected but " + arguments[i].getTypeName() + "\" is found");
            }
            if (!keyOIResolver.update(arguments[i])) {
                throw new UDFArgumentTypeException(i, "Key type \"" + arguments[i].getTypeName() + "\" is different from preceding key types. " + "Previous key type was \"" + arguments[i - 2].getTypeName() + "\"");
            }
        } else {
            // Values
            if (!valueOIResolver.update(arguments[i]) && !compatibleTypes(arguments[i], arguments[i - 2])) {
                throw new UDFArgumentTypeException(i, "Value type \"" + arguments[i].getTypeName() + "\" is different from preceding value types. " + "Previous value type was \"" + arguments[i - 2].getTypeName() + "\"");
            }
        }
    }
    ObjectInspector keyOI = keyOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    ObjectInspector valueOI = valueOIResolver.get(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    converters = new Converter[arguments.length];
    for (int i = 0; i < arguments.length; i++) {
        converters[i] = ObjectInspectorConverters.getConverter(arguments[i], i % 2 == 0 ? keyOI : valueOI);
    }
    return ObjectInspectorFactory.getStandardMapObjectInspector(keyOI, valueOI);
}