Examples with ListObjectInspector org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector used on opensource projects

Example 6 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class MetaStoreUtils method getFieldsFromDeserializer.

/**
   * @param tableName
   * @param deserializer
   * @return the list of fields
   * @throws SerDeException
   * @throws MetaException
   */
public static List<FieldSchema> getFieldsFromDeserializer(String tableName, Deserializer deserializer) throws SerDeException, MetaException {
    ObjectInspector oi = deserializer.getObjectInspector();
    String[] names = tableName.split("\\.");
    String last_name = names[names.length - 1];
    for (int i = 1; i < names.length; i++) {
        if (oi instanceof StructObjectInspector) {
            StructObjectInspector soi = (StructObjectInspector) oi;
            StructField sf = soi.getStructFieldRef(names[i]);
            if (sf == null) {
                throw new MetaException("Invalid Field " + names[i]);
            } else {
                oi = sf.getFieldObjectInspector();
            }
        } else if (oi instanceof ListObjectInspector && names[i].equalsIgnoreCase("$elem$")) {
            ListObjectInspector loi = (ListObjectInspector) oi;
            oi = loi.getListElementObjectInspector();
        } else if (oi instanceof MapObjectInspector && names[i].equalsIgnoreCase("$key$")) {
            MapObjectInspector moi = (MapObjectInspector) oi;
            oi = moi.getMapKeyObjectInspector();
        } else if (oi instanceof MapObjectInspector && names[i].equalsIgnoreCase("$value$")) {
            MapObjectInspector moi = (MapObjectInspector) oi;
            oi = moi.getMapValueObjectInspector();
        } else {
            throw new MetaException("Unknown type for " + names[i]);
        }
    }
    ArrayList<FieldSchema> str_fields = new ArrayList<FieldSchema>();
    // rules on how to recurse the ObjectInspector based on its type
    if (oi.getCategory() != Category.STRUCT) {
        str_fields.add(new FieldSchema(last_name, oi.getTypeName(), FROM_SERIALIZER));
    } else {
        List<? extends StructField> fields = ((StructObjectInspector) oi).getAllStructFieldRefs();
        for (int i = 0; i < fields.size(); i++) {
            StructField structField = fields.get(i);
            String fieldName = structField.getFieldName();
            String fieldTypeName = structField.getFieldObjectInspector().getTypeName();
            String fieldComment = determineFieldComment(structField.getFieldComment());
            str_fields.add(new FieldSchema(fieldName, fieldTypeName, fieldComment));
        }
    }
    return str_fields;
}

Example 7 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class MultiDelimitSerDe method serializeNoEncode.

// This is basically the same as LazySimpleSerDe.serialize. Except that we don't use
// Base64 to encode binary data because we're using printable string as delimiter.
// Consider such a row "strAQ==\1", str is a string, AQ== is the delimiter and \1
// is the binary data.
private static void serializeNoEncode(ByteStream.Output out, Object obj, ObjectInspector objInspector, byte[] separators, int level, Text nullSequence, boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException, SerDeException {
    if (obj == null) {
        out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
        return;
    }
    char separator;
    List<?> list;
    switch(objInspector.getCategory()) {
        case PRIMITIVE:
            PrimitiveObjectInspector oi = (PrimitiveObjectInspector) objInspector;
            if (oi.getPrimitiveCategory() == PrimitiveCategory.BINARY) {
                BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(obj);
                byte[] toWrite = new byte[bw.getLength()];
                System.arraycopy(bw.getBytes(), 0, toWrite, 0, bw.getLength());
                out.write(toWrite, 0, toWrite.length);
            } else {
                LazyUtils.writePrimitiveUTF8(out, obj, oi, escaped, escapeChar, needsEscape);
            }
            return;
        case LIST:
            separator = (char) separators[level];
            ListObjectInspector loi = (ListObjectInspector) objInspector;
            list = loi.getList(obj);
            ObjectInspector eoi = loi.getListElementObjectInspector();
            if (list == null) {
                out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
            } else {
                for (int i = 0; i < list.size(); i++) {
                    if (i > 0) {
                        out.write(separator);
                    }
                    serializeNoEncode(out, list.get(i), eoi, separators, level + 1, nullSequence, escaped, escapeChar, needsEscape);
                }
            }
            return;
        case MAP:
            separator = (char) separators[level];
            char keyValueSeparator = (char) separators[level + 1];
            MapObjectInspector moi = (MapObjectInspector) objInspector;
            ObjectInspector koi = moi.getMapKeyObjectInspector();
            ObjectInspector voi = moi.getMapValueObjectInspector();
            Map<?, ?> map = moi.getMap(obj);
            if (map == null) {
                out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
            } else {
                boolean first = true;
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    if (first) {
                        first = false;
                    } else {
                        out.write(separator);
                    }
                    serializeNoEncode(out, entry.getKey(), koi, separators, level + 2, nullSequence, escaped, escapeChar, needsEscape);
                    out.write(keyValueSeparator);
                    serializeNoEncode(out, entry.getValue(), voi, separators, level + 2, nullSequence, escaped, escapeChar, needsEscape);
                }
            }
            return;
        case STRUCT:
            separator = (char) separators[level];
            StructObjectInspector soi = (StructObjectInspector) objInspector;
            List<? extends StructField> fields = soi.getAllStructFieldRefs();
            list = soi.getStructFieldsDataAsList(obj);
            if (list == null) {
                out.write(nullSequence.getBytes(), 0, nullSequence.getLength());
            } else {
                for (int i = 0; i < list.size(); i++) {
                    if (i > 0) {
                        out.write(separator);
                    }
                    serializeNoEncode(out, list.get(i), fields.get(i).getFieldObjectInspector(), separators, level + 1, nullSequence, escaped, escapeChar, needsEscape);
                }
            }
            return;
    }
    throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
}

Example 8 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class PTFDeserializer method initializeWindowing.

public void initializeWindowing(WindowTableFunctionDef def) throws HiveException {
    ShapeDetails inpShape = def.getInput().getOutputShape();
    /*
     * 1. setup resolve, make connections
     */
    TableFunctionEvaluator tEval = def.getTFunction();
    WindowingTableFunctionResolver tResolver = (WindowingTableFunctionResolver) constructResolver(def.getResolverClassName());
    tResolver.initialize(ptfDesc, def, tEval);
    /*
     * 2. initialize WFns.
     */
    for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
        if (wFnDef.getArgs() != null) {
            for (PTFExpressionDef arg : wFnDef.getArgs()) {
                initialize(arg, inpShape);
            }
        }
        if (wFnDef.getWindowFrame() != null) {
            WindowFrameDef wFrmDef = wFnDef.getWindowFrame();
            initialize(wFrmDef, inpShape);
        }
        setupWdwFnEvaluator(wFnDef);
    }
    ArrayList<String> aliases = new ArrayList<String>();
    ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
    for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
        aliases.add(wFnDef.getAlias());
        if (wFnDef.isPivotResult()) {
            fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
        } else {
            fieldOIs.add(wFnDef.getOI());
        }
    }
    PTFDeserializer.addInputColumnsToList(inpShape, aliases, fieldOIs);
    StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(aliases, fieldOIs);
    tResolver.setWdwProcessingOutputOI(wdwOutOI);
    initialize(def.getOutputShape(), wdwOutOI);
    tResolver.initializeOutputOI();
}

Example 9 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class TestOrcStruct method testInspectorFromTypeInfo.

@Test
public void testInspectorFromTypeInfo() throws Exception {
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString("struct<c1:boolean,c2:tinyint" + ",c3:smallint,c4:int,c5:bigint,c6:float,c7:double,c8:binary," + "c9:string,c10:struct<c1:int>,c11:map<int,int>,c12:uniontype<int>" + ",c13:array<timestamp>>");
    StructObjectInspector inspector = (StructObjectInspector) OrcStruct.createObjectInspector(typeInfo);
    assertEquals("struct<c1:boolean,c2:tinyint,c3:smallint,c4:int,c5:" + "bigint,c6:float,c7:double,c8:binary,c9:string,c10:struct<" + "c1:int>,c11:map<int,int>,c12:uniontype<int>,c13:array<timestamp>>", inspector.getTypeName());
    assertEquals(null, inspector.getAllStructFieldRefs().get(0).getFieldComment());
    assertEquals(null, inspector.getStructFieldRef("UNKNOWN"));
    OrcStruct s1 = new OrcStruct(13);
    for (int i = 0; i < 13; ++i) {
        s1.setFieldValue(i, i);
    }
    List<Object> list = new ArrayList<Object>();
    list.addAll(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12));
    assertEquals(list, inspector.getStructFieldsDataAsList(s1));
    ListObjectInspector listOI = (ListObjectInspector) inspector.getAllStructFieldRefs().get(12).getFieldObjectInspector();
    assertEquals(ObjectInspector.Category.LIST, listOI.getCategory());
    assertEquals(10, listOI.getListElement(list, 10));
    assertEquals(null, listOI.getListElement(list, -1));
    assertEquals(null, listOI.getListElement(list, 13));
    assertEquals(13, listOI.getListLength(list));
    Map<Integer, Integer> map = new HashMap<Integer, Integer>();
    map.put(1, 2);
    map.put(2, 4);
    map.put(3, 6);
    MapObjectInspector mapOI = (MapObjectInspector) inspector.getAllStructFieldRefs().get(10).getFieldObjectInspector();
    assertEquals(3, mapOI.getMapSize(map));
    assertEquals(4, mapOI.getMapValueElement(map, 2));
}

Example 10 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class DynamicSerDeTypeList method serialize.

@Override
public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException, IllegalAccessException {
    ListObjectInspector loi = (ListObjectInspector) oi;
    ObjectInspector elementObjectInspector = loi.getListElementObjectInspector();
    DynamicSerDeTypeBase mt = getElementType();
    WriteNullsProtocol nullProtocol = (oprot instanceof WriteNullsProtocol) ? (WriteNullsProtocol) oprot : null;
    if (o instanceof List) {
        List<?> list = (List<?>) o;
        oprot.writeListBegin(new TList(mt.getType(), list.size()));
        for (Object element : list) {
            if (element == null) {
                assert (nullProtocol != null);
                nullProtocol.writeNull();
            } else {
                mt.serialize(element, elementObjectInspector, oprot);
            }
        }
    } else {
        Object[] list = (Object[]) o;
        oprot.writeListBegin(new TList(mt.getType(), list.length));
        for (Object element : list) {
            if (element == null && nullProtocol != null) {
                assert (nullProtocol != null);
                nullProtocol.writeNull();
            } else {
                mt.serialize(element, elementObjectInspector, oprot);
            }
        }
    }
    // in theory, the below call isn't needed in non thrift_mode, but let's not
    // get too crazy
    oprot.writeListEnd();
}