Examples with MapObjectInspector org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector used on opensource projects

Example 51 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project metacat by Netflix.

the class HiveTypeConverter method getCanonicalType.

/**
 * Returns the canonical type.
 *
 * @param fieldInspector inspector
 * @return type
 */
Type getCanonicalType(final ObjectInspector fieldInspector) {
    switch(fieldInspector.getCategory()) {
        case PRIMITIVE:
            return getPrimitiveType(fieldInspector);
        case MAP:
            final MapObjectInspector mapObjectInspector = TypeUtils.checkType(fieldInspector, MapObjectInspector.class, "fieldInspector");
            final Type keyType = getCanonicalType(mapObjectInspector.getMapKeyObjectInspector());
            final Type valueType = getCanonicalType(mapObjectInspector.getMapValueObjectInspector());
            if (keyType == null || valueType == null) {
                return null;
            }
            return TypeRegistry.getTypeRegistry().getParameterizedType(TypeEnum.MAP, ImmutableList.of(keyType.getTypeSignature(), valueType.getTypeSignature()), ImmutableList.of());
        case LIST:
            final ListObjectInspector listObjectInspector = TypeUtils.checkType(fieldInspector, ListObjectInspector.class, "fieldInspector");
            final Type elementType = getCanonicalType(listObjectInspector.getListElementObjectInspector());
            if (elementType == null) {
                return null;
            }
            return TypeRegistry.getTypeRegistry().getParameterizedType(TypeEnum.ARRAY, ImmutableList.of(elementType.getTypeSignature()), ImmutableList.of());
        case STRUCT:
            final StructObjectInspector structObjectInspector = TypeUtils.checkType(fieldInspector, StructObjectInspector.class, "fieldInspector");
            final List<TypeSignature> fieldTypes = new ArrayList<>();
            final List<Object> fieldNames = new ArrayList<>();
            for (StructField field : structObjectInspector.getAllStructFieldRefs()) {
                fieldNames.add(field.getFieldName());
                final Type fieldType = getCanonicalType(field.getFieldObjectInspector());
                if (fieldType == null) {
                    return null;
                }
                fieldTypes.add(fieldType.getTypeSignature());
            }
            return TypeRegistry.getTypeRegistry().getParameterizedType(TypeEnum.ROW, fieldTypes, fieldNames);
        default:
            log.info("Currently unsupported type {}, returning Unknown type", fieldInspector.getTypeName());
            return BaseType.UNKNOWN;
    }
}

Example 52 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project presto-hive-apache by prestodb.

the class JsonSerDe method buildJSONString.

// TODO : code section copied over from SerDeUtils because of non-standard json production there
// should use quotes for all field names. We should fix this there, and then remove this copy.
// See http://jackson.codehaus.org/1.7.3/javadoc/org/codehaus/jackson/JsonParser.Feature.html#ALLOW_UNQUOTED_FIELD_NAMES
// for details - trying to enable Jackson to ignore that doesn't seem to work(compilation failure
// when attempting to use that feature, so having to change the production itself.
// Also, throws IOException when Binary is detected.
private static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) throws IOException {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                if (o == null) {
                    sb.append("null");
                } else {
                    switch(poi.getPrimitiveCategory()) {
                        case BOOLEAN:
                            {
                                boolean b = ((BooleanObjectInspector) poi).get(o);
                                sb.append(b ? "true" : "false");
                                break;
                            }
                        case BYTE:
                            {
                                sb.append(((ByteObjectInspector) poi).get(o));
                                break;
                            }
                        case SHORT:
                            {
                                sb.append(((ShortObjectInspector) poi).get(o));
                                break;
                            }
                        case INT:
                            {
                                sb.append(((IntObjectInspector) poi).get(o));
                                break;
                            }
                        case LONG:
                            {
                                sb.append(((LongObjectInspector) poi).get(o));
                                break;
                            }
                        case FLOAT:
                            {
                                sb.append(((FloatObjectInspector) poi).get(o));
                                break;
                            }
                        case DOUBLE:
                            {
                                sb.append(((DoubleObjectInspector) poi).get(o));
                                break;
                            }
                        case STRING:
                            {
                                String s = SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
                                appendWithQuotes(sb, s);
                                break;
                            }
                        case BINARY:
                            {
                                throw new IOException("JsonSerDe does not support BINARY type");
                            }
                        case DATE:
                            Date d = ((DateObjectInspector) poi).getPrimitiveJavaObject(o);
                            appendWithQuotes(sb, d.toString());
                            break;
                        case TIMESTAMP:
                            {
                                Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o);
                                appendWithQuotes(sb, t.toString());
                                break;
                            }
                        case DECIMAL:
                            sb.append(((HiveDecimalObjectInspector) poi).getPrimitiveJavaObject(o));
                            break;
                        case VARCHAR:
                            {
                                String s = SerDeUtils.escapeString(((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
                                appendWithQuotes(sb, s);
                                break;
                            }
                        case CHAR:
                            {
                                // this should use HiveChar.getPaddedValue() but it's protected; currently (v0.13)
                                // HiveChar.toString() returns getPaddedValue()
                                String s = SerDeUtils.escapeString(((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
                                appendWithQuotes(sb, s);
                                break;
                            }
                        default:
                            throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
                    }
                }
                break;
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) oi;
                ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
                List<?> olist = loi.getList(o);
                if (olist == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACKET);
                    for (int i = 0; i < olist.size(); i++) {
                        if (i > 0) {
                            sb.append(SerDeUtils.COMMA);
                        }
                        buildJSONString(sb, olist.get(i), listElementObjectInspector);
                    }
                    sb.append(SerDeUtils.RBRACKET);
                }
                break;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) oi;
                ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
                ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
                Map<?, ?> omap = moi.getMap(o);
                if (omap == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    boolean first = true;
                    for (Object entry : omap.entrySet()) {
                        if (first) {
                            first = false;
                        } else {
                            sb.append(SerDeUtils.COMMA);
                        }
                        Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
                        StringBuilder keyBuilder = new StringBuilder();
                        buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
                        String keyString = keyBuilder.toString().trim();
                        if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
                            appendWithQuotes(sb, keyString);
                        } else {
                            sb.append(keyString);
                        }
                        sb.append(SerDeUtils.COLON);
                        buildJSONString(sb, e.getValue(), mapValueObjectInspector);
                    }
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) oi;
                List<? extends StructField> structFields = soi.getAllStructFieldRefs();
                if (o == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    for (int i = 0; i < structFields.size(); i++) {
                        if (i > 0) {
                            sb.append(SerDeUtils.COMMA);
                        }
                        appendWithQuotes(sb, structFields.get(i).getFieldName());
                        sb.append(SerDeUtils.COLON);
                        buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector());
                    }
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) oi;
                if (o == null) {
                    sb.append("null");
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    sb.append(uoi.getTag(o));
                    sb.append(SerDeUtils.COLON);
                    buildJSONString(sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)));
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        default:
            throw new RuntimeException("Unknown type in ObjectInspector!");
    }
}

Example 53 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project presto by prestodb.

the class TestDataWritableWriter method writeValue.

/**
 * It writes the field value to the Parquet RecordConsumer. It detects the field type, and calls
 * the correct write function.
 *
 * @param value The writable object that contains the value.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the type schema.
 */
private void writeValue(final Object value, final ObjectInspector inspector, final Type type) {
    if (type.isPrimitive()) {
        checkInspectorCategory(inspector, ObjectInspector.Category.PRIMITIVE);
        writePrimitive(value, (PrimitiveObjectInspector) inspector);
    } else {
        GroupType groupType = type.asGroupType();
        OriginalType originalType = type.getOriginalType();
        if (originalType != null && originalType.equals(OriginalType.LIST)) {
            checkInspectorCategory(inspector, ObjectInspector.Category.LIST);
            if (singleLevelArray) {
                writeSingleLevelArray(value, (ListObjectInspector) inspector, groupType);
            } else {
                writeArray(value, (ListObjectInspector) inspector, groupType);
            }
        } else if (originalType != null && (originalType.equals(OriginalType.MAP) || originalType.equals(OriginalType.MAP_KEY_VALUE))) {
            checkInspectorCategory(inspector, ObjectInspector.Category.MAP);
            writeMap(value, (MapObjectInspector) inspector, groupType);
        } else {
            checkInspectorCategory(inspector, ObjectInspector.Category.STRUCT);
            writeGroup(value, (StructObjectInspector) inspector, groupType);
        }
    }
}

Example 54 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project presto by prestodb.

the class TestDataWritableWriter method writeMap.

/**
 * It writes a map type and its key-pair values to the Parquet RecordConsumer.
 * This is called when the original type (MAP) is detected by writeValue().
 * This function assumes the following schema:
 * optional group mapCol (MAP) {
 * repeated group map (MAP_KEY_VALUE) {
 * required TYPE key;
 * optional TYPE value;
 * }
 * }
 *
 * @param value The object that contains the map key-values.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the group (MAP) schema.
 */
private void writeMap(final Object value, final MapObjectInspector inspector, final GroupType type) {
    // Get the internal map structure (MAP_KEY_VALUE)
    GroupType repeatedType = type.getType(0).asGroupType();
    recordConsumer.startGroup();
    Map<?, ?> mapValues = inspector.getMap(value);
    if (mapValues != null && mapValues.size() > 0) {
        recordConsumer.startField(repeatedType.getName(), 0);
        Type keyType = repeatedType.getType(0);
        String keyName = keyType.getName();
        ObjectInspector keyInspector = inspector.getMapKeyObjectInspector();
        Type valuetype = repeatedType.getType(1);
        String valueName = valuetype.getName();
        ObjectInspector valueInspector = inspector.getMapValueObjectInspector();
        for (Map.Entry<?, ?> keyValue : mapValues.entrySet()) {
            recordConsumer.startGroup();
            if (keyValue != null) {
                // write key element
                Object keyElement = keyValue.getKey();
                recordConsumer.startField(keyName, 0);
                writeValue(keyElement, keyInspector, keyType);
                recordConsumer.endField(keyName, 0);
                // write value element
                Object valueElement = keyValue.getValue();
                if (valueElement != null) {
                    recordConsumer.startField(valueName, 1);
                    writeValue(valueElement, valueInspector, valuetype);
                    recordConsumer.endField(valueName, 1);
                }
            }
            recordConsumer.endGroup();
        }
        recordConsumer.endField(repeatedType.getName(), 0);
    }
    recordConsumer.endGroup();
}

Example 55 with MapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector in project flink by apache.

the class HiveInspectors method getConversion.

/**
 * Get conversion for converting Flink object to Hive object from an ObjectInspector and the
 * corresponding Flink DataType.
 */
public static HiveObjectConversion getConversion(ObjectInspector inspector, LogicalType dataType, HiveShim hiveShim) {
    if (inspector instanceof PrimitiveObjectInspector) {
        HiveObjectConversion conversion;
        if (inspector instanceof BooleanObjectInspector || inspector instanceof StringObjectInspector || inspector instanceof ByteObjectInspector || inspector instanceof ShortObjectInspector || inspector instanceof IntObjectInspector || inspector instanceof LongObjectInspector || inspector instanceof FloatObjectInspector || inspector instanceof DoubleObjectInspector || inspector instanceof BinaryObjectInspector || inspector instanceof VoidObjectInspector) {
            conversion = IdentityConversion.INSTANCE;
        } else if (inspector instanceof DateObjectInspector) {
            conversion = hiveShim::toHiveDate;
        } else if (inspector instanceof TimestampObjectInspector) {
            conversion = hiveShim::toHiveTimestamp;
        } else if (inspector instanceof HiveCharObjectInspector) {
            conversion = o -> o == null ? null : new HiveChar((String) o, ((CharType) dataType).getLength());
        } else if (inspector instanceof HiveVarcharObjectInspector) {
            conversion = o -> o == null ? null : new HiveVarchar((String) o, ((VarCharType) dataType).getLength());
        } else if (inspector instanceof HiveDecimalObjectInspector) {
            conversion = o -> o == null ? null : HiveDecimal.create((BigDecimal) o);
        } else {
            throw new FlinkHiveUDFException("Unsupported primitive object inspector " + inspector.getClass().getName());
        }
        // currently this happens for constant arguments for UDFs
        if (((PrimitiveObjectInspector) inspector).preferWritable()) {
            conversion = new WritableHiveObjectConversion(conversion, hiveShim);
        }
        return conversion;
    }
    if (inspector instanceof ListObjectInspector) {
        HiveObjectConversion eleConvert = getConversion(((ListObjectInspector) inspector).getListElementObjectInspector(), ((ArrayType) dataType).getElementType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Object[] array = (Object[]) o;
            List<Object> result = new ArrayList<>();
            for (Object ele : array) {
                result.add(eleConvert.toHiveObject(ele));
            }
            return result;
        };
    }
    if (inspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) inspector;
        MapType kvType = (MapType) dataType;
        HiveObjectConversion keyConversion = getConversion(mapInspector.getMapKeyObjectInspector(), kvType.getKeyType(), hiveShim);
        HiveObjectConversion valueConversion = getConversion(mapInspector.getMapValueObjectInspector(), kvType.getValueType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Map<Object, Object> map = (Map) o;
            Map<Object, Object> result = new HashMap<>(map.size());
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                result.put(keyConversion.toHiveObject(entry.getKey()), valueConversion.toHiveObject(entry.getValue()));
            }
            return result;
        };
    }
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector structInspector = (StructObjectInspector) inspector;
        List<? extends StructField> structFields = structInspector.getAllStructFieldRefs();
        List<RowType.RowField> rowFields = ((RowType) dataType).getFields();
        HiveObjectConversion[] conversions = new HiveObjectConversion[structFields.size()];
        for (int i = 0; i < structFields.size(); i++) {
            conversions[i] = getConversion(structFields.get(i).getFieldObjectInspector(), rowFields.get(i).getType(), hiveShim);
        }
        return o -> {
            if (o == null) {
                return null;
            }
            Row row = (Row) o;
            List<Object> result = new ArrayList<>(row.getArity());
            for (int i = 0; i < row.getArity(); i++) {
                result.add(conversions[i].toHiveObject(row.getField(i)));
            }
            return result;
        };
    }
    throw new FlinkHiveUDFException(String.format("Flink doesn't support convert object conversion for %s yet", inspector));
}