Examples with StructField org.apache.hadoop.hive.serde2.objectinspector.StructField used on opensource projects

Example 11 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class ObjectInspectorUtils method compareTypes.

/**
   * Compares two types identified by the given object inspectors. This method
   * compares the types as follows:
   * <ol>
   * <li>If the given inspectors do not belong to same category, the result is
   * negative.</li>
   * <li>If the given inspectors are for <code>PRIMITIVE</code> type, the result
   * is the comparison of their type names.</li>
   * <li>If the given inspectors are for <code>LIST</code> type, then the result
   * is recursive call to compare the type of list elements.</li>
   * <li>If the given inspectors are <code>MAP</code> type, then the result is a
   * recursive call to compare the map key and value types.</li>
   * <li>If the given inspectors are <code>STRUCT</code> type, then the result
   * is negative if they do not have the same number of fields. If they do have
   * the same number of fields, the result is a recursive call to compare each
   * of the field types.</li>
   * <li>If none of the above, the result is negative.</li>
   * </ol>
   * @param o1
   * @param o2
   * @return true if the given object inspectors represent the same types.
   */
public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) {
    Category c1 = o1.getCategory();
    Category c2 = o2.getCategory();
    // Return false if categories are not equal
    if (!c1.equals(c2)) {
        return false;
    }
    // If both categories are primitive return the comparison of type names.
    if (c1.equals(Category.PRIMITIVE)) {
        return o1.getTypeName().equals(o2.getTypeName());
    }
    // If lists, recursively compare the list element types
    if (c1.equals(Category.LIST)) {
        ObjectInspector child1 = ((ListObjectInspector) o1).getListElementObjectInspector();
        ObjectInspector child2 = ((ListObjectInspector) o2).getListElementObjectInspector();
        return compareTypes(child1, child2);
    }
    // If maps, recursively compare the key and value types
    if (c1.equals(Category.MAP)) {
        MapObjectInspector mapOI1 = (MapObjectInspector) o1;
        MapObjectInspector mapOI2 = (MapObjectInspector) o2;
        ObjectInspector childKey1 = mapOI1.getMapKeyObjectInspector();
        ObjectInspector childKey2 = mapOI2.getMapKeyObjectInspector();
        if (compareTypes(childKey1, childKey2)) {
            ObjectInspector childVal1 = mapOI1.getMapValueObjectInspector();
            ObjectInspector childVal2 = mapOI2.getMapValueObjectInspector();
            if (compareTypes(childVal1, childVal2)) {
                return true;
            }
        }
        return false;
    }
    // If structs, recursively compare the fields
    if (c1.equals(Category.STRUCT)) {
        StructObjectInspector structOI1 = (StructObjectInspector) o1;
        StructObjectInspector structOI2 = (StructObjectInspector) o2;
        List<? extends StructField> childFieldsList1 = structOI1.getAllStructFieldRefs();
        List<? extends StructField> childFieldsList2 = structOI2.getAllStructFieldRefs();
        if (childFieldsList1 == null && childFieldsList2 == null) {
            return true;
        } else if (childFieldsList1 == null || childFieldsList2 == null) {
            return false;
        } else if (childFieldsList1.size() != childFieldsList2.size()) {
            return false;
        }
        Iterator<? extends StructField> it1 = childFieldsList1.iterator();
        Iterator<? extends StructField> it2 = childFieldsList2.iterator();
        while (it1.hasNext()) {
            StructField field1 = it1.next();
            StructField field2 = it2.next();
            if (!compareTypes(field1.getFieldObjectInspector(), field2.getFieldObjectInspector())) {
                return false;
            }
        }
        return true;
    }
    if (c1.equals(Category.UNION)) {
        UnionObjectInspector uoi1 = (UnionObjectInspector) o1;
        UnionObjectInspector uoi2 = (UnionObjectInspector) o2;
        List<ObjectInspector> ois1 = uoi1.getObjectInspectors();
        List<ObjectInspector> ois2 = uoi2.getObjectInspectors();
        if (ois1 == null && ois2 == null) {
            return true;
        } else if (ois1 == null || ois2 == null) {
            return false;
        } else if (ois1.size() != ois2.size()) {
            return false;
        }
        Iterator<? extends ObjectInspector> it1 = ois1.iterator();
        Iterator<? extends ObjectInspector> it2 = ois2.iterator();
        while (it1.hasNext()) {
            if (!compareTypes(it1.next(), it2.next())) {
                return false;
            }
        }
        return true;
    }
    // Unknown category
    throw new RuntimeException("Unknown category encountered: " + c1);
}

Example 12 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class ThriftFormatter method convert.

@Override
public Object convert(Object row, ObjectInspector rowOI) throws Exception {
    StructObjectInspector structOI = (StructObjectInspector) rowOI;
    List<? extends StructField> fields = structOI.getAllStructFieldRefs();
    Object[] converted = new Object[fields.size()];
    for (int i = 0; i < converted.length; i++) {
        StructField fieldRef = fields.get(i);
        Object field = structOI.getStructFieldData(row, fieldRef);
        converted[i] = field == null ? null : SerDeUtils.toThriftPayload(field, fieldRef.getFieldObjectInspector(), protocol);
    }
    return converted;
}

Example 13 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class LazySimpleStructObjectInspector method getStructFieldData.

// With Data
@Override
public Object getStructFieldData(Object data, StructField fieldRef) {
    if (data == null) {
        return null;
    }
    StructObject struct = (StructObject) data;
    MyField f = (MyField) fieldRef;
    int fieldID = f.getFieldID();
    assert (fieldID >= 0 && fieldID < fields.size());
    ObjectInspector oi = f.getFieldObjectInspector();
    if (oi instanceof AvroLazyObjectInspector) {
        return ((AvroLazyObjectInspector) oi).getStructFieldData(data, fieldRef);
    }
    if (oi instanceof MapObjectInspector) {
        ObjectInspector valueOI = ((MapObjectInspector) oi).getMapValueObjectInspector();
        if (valueOI instanceof AvroLazyObjectInspector) {
            return ((AvroLazyObjectInspector) valueOI).getStructFieldData(data, fieldRef);
        }
    }
    return struct.getField(fieldID);
}

Example 14 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class TestObjectInspectorUtils method testObjectInspectorUtils.

public void testObjectInspectorUtils() throws Throwable {
    try {
        ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        // metadata
        assertEquals(Category.STRUCT, oi1.getCategory());
        // standard ObjectInspector
        StructObjectInspector soi = (StructObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(oi1);
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        assertEquals(10, fields.size());
        assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
        // null
        for (int i = 0; i < fields.size(); i++) {
            assertNull(soi.getStructFieldData(null, fields.get(i)));
        }
        // real object
        Complex cc = new Complex();
        cc.setAint(1);
        cc.setAString("test");
        List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
        cc.setLint(c2);
        List<String> c3 = Arrays.asList(new String[] { "one", "two" });
        cc.setLString(c3);
        List<IntString> c4 = new ArrayList<IntString>();
        cc.setLintString(c4);
        cc.setMStringString(null);
        // standard object
        Object c = ObjectInspectorUtils.copyToStandardObject(cc, oi1);
        assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
        assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
        assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
        assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
        assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
        assertNull(soi.getStructFieldData(c, fields.get(5)));
        ArrayList<Object> cfields = new ArrayList<Object>();
        for (int i = 0; i < 10; i++) {
            cfields.add(soi.getStructFieldData(c, fields.get(i)));
        }
        assertEquals(cfields, soi.getStructFieldsDataAsList(c));
        // sub fields
        assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
        assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
        assertEquals(ObjectInspectorUtils.getStandardObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT))), fields.get(4).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(5).getFieldObjectInspector());
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 15 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class TestProtocolBuffersObjectInspectors method testProtocolBuffersObjectInspectors.

public void testProtocolBuffersObjectInspectors() throws Throwable {
    try {
        ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS);
        ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS);
        assertEquals(oi1, oi2);
        // metadata
        assertEquals(Category.STRUCT, oi1.getCategory());
        StructObjectInspector soi = (StructObjectInspector) oi1;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        assertEquals(5, fields.size());
        assertEquals(fields.get(0), soi.getStructFieldRef("aint_"));
        // null
        for (int i = 0; i < fields.size(); i++) {
            assertNull(soi.getStructFieldData(null, fields.get(i)));
        }
        // real object
        List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
        List<String> c3 = Arrays.asList(new String[] { "one", "two" });
        List<IntString> c4 = new ArrayList<IntString>();
        Complex c = new Complex(1, "test", c2, c3, c4);
        assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
        assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
        assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
        assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
        assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
        ArrayList<Object> cfields = new ArrayList<Object>();
        for (int i = 0; i < 5; i++) {
            cfields.add(soi.getStructFieldData(c, fields.get(i)));
        }
        assertEquals(cfields, soi.getStructFieldsDataAsList(c));
        // sub fields
        assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
        assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS)), fields.get(4).getFieldObjectInspector());
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}