Examples with ListObjectInspector org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector used on opensource projects

Example 16 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class PTFTranslator method buildRowResolverForWindowing.

protected RowResolver buildRowResolverForWindowing(WindowTableFunctionDef def) throws SemanticException {
    RowResolver rr = new RowResolver();
    HashMap<String, WindowExpressionSpec> aliasToExprMap = windowingSpec.getAliasToWdwExpr();
    /*
     * add Window Functions
     */
    for (WindowFunctionDef wFnDef : def.getWindowFunctions()) {
        ASTNode ast = aliasToExprMap.get(wFnDef.getAlias()).getExpression();
        ObjectInspector wFnOI = null;
        if (wFnDef.isPivotResult()) {
            wFnOI = ((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector();
        } else {
            wFnOI = wFnDef.getOI();
        }
        ColumnInfo cInfo = new ColumnInfo(wFnDef.getAlias(), TypeInfoUtils.getTypeInfoFromObjectInspector(wFnOI), null, true, true);
        rr.putExpression(ast, cInfo);
    }
    RowResolver inpRR = def.getRawInputShape().getRr();
    /*
     * add columns from inpRR
     */
    for (ColumnInfo inpCInfo : inputRR.getColumnInfos()) {
        ColumnInfo cInfo = new ColumnInfo(inpCInfo);
        ASTNode inExpr = PTFTranslator.getASTNode(inpCInfo, inpRR);
        if (inExpr != null) {
            rr.putExpression(inExpr, cInfo);
        } else {
            String[] tabColAlias = inputRR.reverseLookup(inpCInfo.getInternalName());
            if (tabColAlias != null) {
                rr.put(tabColAlias[0], tabColAlias[1], cInfo);
            } else {
                rr.put(inpCInfo.getTabAlias(), inpCInfo.getAlias(), cInfo);
            }
        }
        String[] altMapping = inputRR.getAlternateMappings(inpCInfo.getInternalName());
        if (altMapping != null) {
            rr.put(altMapping[0], altMapping[1], cInfo);
        }
    }
    return rr;
}

Example 17 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class AccumuloRowSerializer method writeWithLevel.

/**
   * Recursively serialize an Object using its {@link ObjectInspector}, respecting the
   * separators defined by the {@link LazySerDeParameters}.
   * @param oi ObjectInspector for the current object
   * @param value The current object
   * @param output A buffer output is written to
   * @param mapping The mapping for this Hive column
   * @param level The current level/offset for the SerDe separator
   * @throws IOException
   */
protected void writeWithLevel(ObjectInspector oi, Object value, ByteStream.Output output, ColumnMapping mapping, int level) throws IOException {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            if (mapping.getEncoding() == ColumnEncoding.BINARY) {
                this.writeBinary(output, value, (PrimitiveObjectInspector) oi);
            } else {
                this.writeString(output, value, (PrimitiveObjectInspector) oi);
            }
            return;
        case LIST:
            char separator = (char) serDeParams.getSeparators()[level];
            ListObjectInspector loi = (ListObjectInspector) oi;
            List<?> list = loi.getList(value);
            ObjectInspector eoi = loi.getListElementObjectInspector();
            if (list == null) {
                log.debug("No objects found when serializing list");
                return;
            } else {
                for (int i = 0; i < list.size(); i++) {
                    if (i > 0) {
                        output.write(separator);
                    }
                    writeWithLevel(eoi, list.get(i), output, mapping, level + 1);
                }
            }
            return;
        case MAP:
            char sep = (char) serDeParams.getSeparators()[level];
            char keyValueSeparator = (char) serDeParams.getSeparators()[level + 1];
            MapObjectInspector moi = (MapObjectInspector) oi;
            ObjectInspector koi = moi.getMapKeyObjectInspector();
            ObjectInspector voi = moi.getMapValueObjectInspector();
            Map<?, ?> map = moi.getMap(value);
            if (map == null) {
                log.debug("No object found when serializing map");
                return;
            } else {
                boolean first = true;
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    if (first) {
                        first = false;
                    } else {
                        output.write(sep);
                    }
                    writeWithLevel(koi, entry.getKey(), output, mapping, level + 2);
                    output.write(keyValueSeparator);
                    writeWithLevel(voi, entry.getValue(), output, mapping, level + 2);
                }
            }
            return;
        case STRUCT:
            sep = (char) serDeParams.getSeparators()[level];
            StructObjectInspector soi = (StructObjectInspector) oi;
            List<? extends StructField> fields = soi.getAllStructFieldRefs();
            list = soi.getStructFieldsDataAsList(value);
            if (list == null) {
                log.debug("No object found when serializing struct");
                return;
            } else {
                for (int i = 0; i < list.size(); i++) {
                    if (i > 0) {
                        output.write(sep);
                    }
                    writeWithLevel(fields.get(i).getFieldObjectInspector(), list.get(i), output, mapping, level + 1);
                }
            }
            return;
        default:
            throw new RuntimeException("Unknown category type: " + oi.getCategory());
    }
}

Example 18 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class HCatRecordSerDe method serializeList.

private static List<?> serializeList(Object f, ListObjectInspector loi) throws SerDeException {
    List l = loi.getList(f);
    if (l == null) {
        return null;
    }
    ObjectInspector eloi = loi.getListElementObjectInspector();
    if (eloi.getCategory() == Category.PRIMITIVE) {
        List<Object> list = new ArrayList<Object>(l.size());
        for (int i = 0; i < l.size(); i++) {
            list.add(((PrimitiveObjectInspector) eloi).getPrimitiveJavaObject(l.get(i)));
        }
        return list;
    } else if (eloi.getCategory() == Category.STRUCT) {
        List<List<?>> list = new ArrayList<List<?>>(l.size());
        for (int i = 0; i < l.size(); i++) {
            list.add(serializeStruct(l.get(i), (StructObjectInspector) eloi));
        }
        return list;
    } else if (eloi.getCategory() == Category.LIST) {
        List<List<?>> list = new ArrayList<List<?>>(l.size());
        for (int i = 0; i < l.size(); i++) {
            list.add(serializeList(l.get(i), (ListObjectInspector) eloi));
        }
        return list;
    } else if (eloi.getCategory() == Category.MAP) {
        List<Map<?, ?>> list = new ArrayList<Map<?, ?>>(l.size());
        for (int i = 0; i < l.size(); i++) {
            list.add(serializeMap(l.get(i), (MapObjectInspector) eloi));
        }
        return list;
    } else {
        throw new SerDeException(HCatRecordSerDe.class.toString() + " does not know what to do with fields of unknown category: " + eloi.getCategory() + " , type: " + eloi.getTypeName());
    }
}

Example 19 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class JsonSerDe method buildJSONString.

// TODO : code section copied over from SerDeUtils because of non-standard json production there
// should use quotes for all field names. We should fix this there, and then remove this copy.
// See http://jackson.codehaus.org/1.7.3/javadoc/org/codehaus/jackson/JsonParser.Feature.html#ALLOW_UNQUOTED_FIELD_NAMES
// for details - trying to enable Jackson to ignore that doesn't seem to work(compilation failure
// when attempting to use that feature, so having to change the production itself.
// Also, throws IOException when Binary is detected.
private static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) throws IOException {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                if (o == null) {
                    sb.append("null");
                } else {
                    switch(poi.getPrimitiveCategory()) {
                        case BOOLEAN:
                            {
                                boolean b = ((BooleanObjectInspector) poi).get(o);
                                sb.append(b ? "true" : "false");
                                break;
                            }
                        case BYTE:
                            {
                                sb.append(((ByteObjectInspector) poi).get(o));
                                break;
                            }
                        case SHORT:
                            {
                                sb.append(((ShortObjectInspector) poi).get(o));
                                break;
                            }
                        case INT:
                            {
                                sb.append(((IntObjectInspector) poi).get(o));
                                break;
                            }
                        case LONG:
                            {
                                sb.append(((LongObjectInspector) poi).get(o));
                                break;
                            }
                        case FLOAT:
                            {
                                sb.append(((FloatObjectInspector) poi).get(o));
                                break;
                            }
                        case DOUBLE:
                            {
                                sb.append(((DoubleObjectInspector) poi).get(o));
                                break;
                            }
                        case STRING:
                            {
                                String s = SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
                                appendWithQuotes(sb, s);
                                break;
                            }
                        case BINARY:
                            {
                                throw new IOException("JsonSerDe does not support BINARY type");
                            }
                        case DATE:
                            Date d = ((DateObjectInspector) poi).getPrimitiveJavaObject(o);
                            appendWithQuotes(sb, d.toString());
                            break;
                        case TIMESTAMP:
                            {
                                Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o);
                                appendWithQuotes(sb, t.toString());
                                break;
                            }
                        case DECIMAL:
                            sb.append(((HiveDecimalObjectInspector) poi).getPrimitiveJavaObject(o));
                            break;
                        case VARCHAR:
                            {
                                String s = SerDeUtils.escapeString(((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
                                appendWithQuotes(sb, s);
                                break;
                            }
                        case CHAR:
                            {
                                //this should use HiveChar.getPaddedValue() but it's protected; currently (v0.13)
                                // HiveChar.toString() returns getPaddedValue()
                                String s = SerDeUtils.escapeString(((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
                                appendWithQuotes(sb, s);
                                break;
                            }
                        default:
                            throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
                    }
                }
                break;
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) oi;
                ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
                List<?> olist = loi.getList(o);
                if (olist == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACKET);
                    for (int i = 0; i < olist.size(); i++) {
                        if (i > 0) {
                            sb.append(SerDeUtils.COMMA);
                        }
                        buildJSONString(sb, olist.get(i), listElementObjectInspector);
                    }
                    sb.append(SerDeUtils.RBRACKET);
                }
                break;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) oi;
                ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
                ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
                Map<?, ?> omap = moi.getMap(o);
                if (omap == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    boolean first = true;
                    for (Object entry : omap.entrySet()) {
                        if (first) {
                            first = false;
                        } else {
                            sb.append(SerDeUtils.COMMA);
                        }
                        Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
                        StringBuilder keyBuilder = new StringBuilder();
                        buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
                        String keyString = keyBuilder.toString().trim();
                        if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
                            appendWithQuotes(sb, keyString);
                        } else {
                            sb.append(keyString);
                        }
                        sb.append(SerDeUtils.COLON);
                        buildJSONString(sb, e.getValue(), mapValueObjectInspector);
                    }
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) oi;
                List<? extends StructField> structFields = soi.getAllStructFieldRefs();
                if (o == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    for (int i = 0; i < structFields.size(); i++) {
                        if (i > 0) {
                            sb.append(SerDeUtils.COMMA);
                        }
                        appendWithQuotes(sb, structFields.get(i).getFieldName());
                        sb.append(SerDeUtils.COLON);
                        buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector());
                    }
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) oi;
                if (o == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    sb.append(uoi.getTag(o));
                    sb.append(SerDeUtils.COLON);
                    buildJSONString(sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        default:
            throw new RuntimeException("Unknown type in ObjectInspector!");
    }
}

Example 20 with ListObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector in project hive by apache.

the class InternalUtil method getObjectInspector.

private static ObjectInspector getObjectInspector(TypeInfo type) throws IOException {
    switch(type.getCategory()) {
        case PRIMITIVE:
            PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
            return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
        case MAP:
            MapTypeInfo mapType = (MapTypeInfo) type;
            MapObjectInspector mapInspector = ObjectInspectorFactory.getStandardMapObjectInspector(getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
            return mapInspector;
        case LIST:
            ListTypeInfo listType = (ListTypeInfo) type;
            ListObjectInspector listInspector = ObjectInspectorFactory.getStandardListObjectInspector(getObjectInspector(listType.getListElementTypeInfo()));
            return listInspector;
        case STRUCT:
            StructTypeInfo structType = (StructTypeInfo) type;
            List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
            List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
            for (TypeInfo fieldType : fieldTypes) {
                fieldInspectors.add(getObjectInspector(fieldType));
            }
            StructObjectInspector structInspector = ObjectInspectorFactory.getStandardStructObjectInspector(structType.getAllStructFieldNames(), fieldInspectors);
            return structInspector;
        default:
            throw new IOException("Unknown field schema type");
    }
}