Examples with DateObjectInspector org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector used on opensource projects

Example 36 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project azkaban by azkaban.

the class SerDeUtilsWrapper method buildJSONString.

private static void buildJSONString(StringBuilder sb, Object obj, ObjectInspector objIns) {
    String nullStr = "null";
    switch(objIns.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objIns;
                if (obj == null) {
                    sb.append(nullStr);
                } else {
                    switch(poi.getPrimitiveCategory()) {
                        case BOOLEAN:
                            {
                                boolean b = ((BooleanObjectInspector) poi).get(obj);
                                sb.append(b ? "true" : "false");
                                break;
                            }
                        case BYTE:
                            {
                                sb.append(((ByteObjectInspector) poi).get(obj));
                                break;
                            }
                        case SHORT:
                            {
                                sb.append(((ShortObjectInspector) poi).get(obj));
                                break;
                            }
                        case INT:
                            {
                                sb.append(((IntObjectInspector) poi).get(obj));
                                break;
                            }
                        case LONG:
                            {
                                sb.append(((LongObjectInspector) poi).get(obj));
                                break;
                            }
                        case FLOAT:
                            {
                                sb.append(((FloatObjectInspector) poi).get(obj));
                                break;
                            }
                        case DOUBLE:
                            {
                                sb.append(((DoubleObjectInspector) poi).get(obj));
                                break;
                            }
                        case STRING:
                            {
                                sb.append('"');
                                sb.append(SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(obj)));
                                sb.append('"');
                                break;
                            }
                        case VARCHAR:
                            {
                                sb.append('"');
                                sb.append(SerDeUtils.escapeString(((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(obj).toString()));
                                sb.append('"');
                                break;
                            }
                        case DATE:
                            {
                                sb.append('"');
                                sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(obj));
                                sb.append('"');
                                break;
                            }
                        case TIMESTAMP:
                            {
                                sb.append('"');
                                sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(obj));
                                sb.append('"');
                                break;
                            }
                        case BINARY:
                            {
                                BytesWritable bw = ((BinaryObjectInspector) objIns).getPrimitiveWritableObject(obj);
                                Text txt = new Text();
                                txt.set(bw.getBytes(), 0, bw.getLength());
                                // Fix to serialize binary type
                                sb.append('"');
                                sb.append(StringEscapeUtils.escapeJava(txt.toString()));
                                sb.append('"');
                                break;
                            }
                        case DECIMAL:
                            {
                                sb.append(((HiveDecimalObjectInspector) objIns).getPrimitiveJavaObject(obj));
                                break;
                            }
                        default:
                            throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
                    }
                }
                break;
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) objIns;
                ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
                List<?> olist = loi.getList(obj);
                if (olist == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(SerDeUtils.LBRACKET);
                    for (int i = 0; i < olist.size(); i++) {
                        if (i > 0) {
                            sb.append(SerDeUtils.COMMA);
                        }
                        buildJSONString(sb, olist.get(i), listElementObjectInspector);
                    }
                    sb.append(SerDeUtils.RBRACKET);
                }
                break;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) objIns;
                ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
                ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
                Map<?, ?> omap = moi.getMap(obj);
                if (omap == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    boolean first = true;
                    for (Object entry : omap.entrySet()) {
                        if (first) {
                            first = false;
                        } else {
                            sb.append(SerDeUtils.COMMA);
                        }
                        Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
                        buildJSONString(sb, e.getKey(), mapKeyObjectInspector);
                        sb.append(SerDeUtils.COLON);
                        buildJSONString(sb, e.getValue(), mapValueObjectInspector);
                    }
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) objIns;
                List<? extends StructField> structFields = soi.getAllStructFieldRefs();
                if (obj == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    for (int i = 0; i < structFields.size(); i++) {
                        if (i > 0) {
                            sb.append(SerDeUtils.COMMA);
                        }
                        sb.append(SerDeUtils.QUOTE);
                        sb.append(structFields.get(i).getFieldName());
                        sb.append(SerDeUtils.QUOTE);
                        sb.append(SerDeUtils.COLON);
                        buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)), structFields.get(i).getFieldObjectInspector());
                    }
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) objIns;
                if (obj == null) {
                    sb.append(nullStr);
                } else {
                    sb.append(SerDeUtils.LBRACE);
                    sb.append(uoi.getTag(obj));
                    sb.append(SerDeUtils.COLON);
                    buildJSONString(sb, uoi.getField(obj), uoi.getObjectInspectors().get(uoi.getTag(obj)));
                    sb.append(SerDeUtils.RBRACE);
                }
                break;
            }
        default:
            throw new RuntimeException("Unknown type in ObjectInspector!");
    }
}

Example 37 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project presto by prestodb.

the class SerDeUtils method serializePrimitive.

private static void serializePrimitive(Type type, BlockBuilder builder, Object object, PrimitiveObjectInspector inspector) {
    requireNonNull(builder, "parent builder is null");
    if (object == null) {
        builder.appendNull();
        return;
    }
    switch(inspector.getPrimitiveCategory()) {
        case BOOLEAN:
            BooleanType.BOOLEAN.writeBoolean(builder, ((BooleanObjectInspector) inspector).get(object));
            return;
        case BYTE:
            TinyintType.TINYINT.writeLong(builder, ((ByteObjectInspector) inspector).get(object));
            return;
        case SHORT:
            SmallintType.SMALLINT.writeLong(builder, ((ShortObjectInspector) inspector).get(object));
            return;
        case INT:
            IntegerType.INTEGER.writeLong(builder, ((IntObjectInspector) inspector).get(object));
            return;
        case LONG:
            BigintType.BIGINT.writeLong(builder, ((LongObjectInspector) inspector).get(object));
            return;
        case FLOAT:
            RealType.REAL.writeLong(builder, floatToRawIntBits(((FloatObjectInspector) inspector).get(object)));
            return;
        case DOUBLE:
            DoubleType.DOUBLE.writeDouble(builder, ((DoubleObjectInspector) inspector).get(object));
            return;
        case STRING:
            type.writeSlice(builder, Slices.utf8Slice(((StringObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case VARCHAR:
            type.writeSlice(builder, Slices.utf8Slice(((HiveVarcharObjectInspector) inspector).getPrimitiveJavaObject(object).getValue()));
            return;
        case CHAR:
            CharType charType = (CharType) type;
            HiveChar hiveChar = ((HiveCharObjectInspector) inspector).getPrimitiveJavaObject(object);
            type.writeSlice(builder, truncateToLengthAndTrimSpaces(Slices.utf8Slice(hiveChar.getValue()), charType.getLength()));
            return;
        case DATE:
            DateType.DATE.writeLong(builder, formatDateAsLong(object, (DateObjectInspector) inspector));
            return;
        case TIMESTAMP:
            TimestampType.TIMESTAMP.writeLong(builder, formatTimestampAsLong(object, (TimestampObjectInspector) inspector));
            return;
        case BINARY:
            VARBINARY.writeSlice(builder, Slices.wrappedBuffer(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object)));
            return;
        case DECIMAL:
            DecimalType decimalType = (DecimalType) type;
            HiveDecimalWritable hiveDecimal = ((HiveDecimalObjectInspector) inspector).getPrimitiveWritableObject(object);
            if (decimalType.isShort()) {
                decimalType.writeLong(builder, DecimalUtils.getShortDecimalValue(hiveDecimal, decimalType.getScale()));
            } else {
                decimalType.writeSlice(builder, DecimalUtils.getLongDecimalValue(hiveDecimal, decimalType.getScale()));
            }
            return;
    }
    throw new RuntimeException("Unknown primitive type: " + inspector.getPrimitiveCategory());
}

Example 38 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project incubator-linkis by apache.

the class CustomerDelimitedJSONSerDe method writePrimitiveUTF8.

private static void writePrimitiveUTF8(OutputStream out, Object o, PrimitiveObjectInspector oi, boolean escaped, byte escapeChar, boolean[] needsEscape) throws IOException {
    PrimitiveObjectInspector.PrimitiveCategory category = oi.getPrimitiveCategory();
    byte[] binaryData = null;
    WritableComparable wc = null;
    switch(category) {
        case BOOLEAN:
            {
                boolean b = ((BooleanObjectInspector) oi).get(o);
                if (b) {
                    binaryData = Base64.encodeBase64(trueBytes);
                } else {
                    binaryData = Base64.encodeBase64(falseBytes);
                }
                break;
            }
        case BYTE:
            {
                binaryData = Base64.encodeBase64(String.valueOf(((ByteObjectInspector) oi).get(o)).getBytes());
                break;
            }
        case SHORT:
            {
                binaryData = Base64.encodeBase64(String.valueOf(((ShortObjectInspector) oi).get(o)).getBytes());
                break;
            }
        case INT:
            {
                binaryData = Base64.encodeBase64(String.valueOf(((IntObjectInspector) oi).get(o)).getBytes());
                break;
            }
        case LONG:
            {
                binaryData = Base64.encodeBase64(String.valueOf(((LongObjectInspector) oi).get(o)).getBytes());
                break;
            }
        case FLOAT:
            {
                binaryData = Base64.encodeBase64(String.valueOf(((FloatObjectInspector) oi).get(o)).getBytes());
                break;
            }
        case DOUBLE:
            {
                binaryData = Base64.encodeBase64(String.valueOf(((DoubleObjectInspector) oi).get(o)).getBytes());
                break;
            }
        case STRING:
            {
                binaryData = Base64.encodeBase64(((StringObjectInspector) oi).getPrimitiveWritableObject(o).getBytes());
                break;
            }
        case CHAR:
            {
                HiveCharWritable hc = ((HiveCharObjectInspector) oi).getPrimitiveWritableObject(o);
                binaryData = Base64.encodeBase64(String.valueOf(hc).getBytes());
                break;
            }
        case VARCHAR:
            {
                HiveVarcharWritable hc = ((HiveVarcharObjectInspector) oi).getPrimitiveWritableObject(o);
                binaryData = Base64.encodeBase64(String.valueOf(hc).getBytes());
                break;
            }
        case BINARY:
            {
                BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
                byte[] toEncode = new byte[bw.getLength()];
                System.arraycopy(bw.getBytes(), 0, toEncode, 0, bw.getLength());
                byte[] toWrite = Base64.encodeBase64(toEncode);
                out.write(toWrite, 0, toWrite.length);
                break;
            }
        case DATE:
            {
                wc = ((DateObjectInspector) oi).getPrimitiveWritableObject(o);
                binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes());
                break;
            }
        case TIMESTAMP:
            {
                wc = ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o);
                binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes());
                break;
            }
        case INTERVAL_YEAR_MONTH:
            {
                wc = ((HiveIntervalYearMonthObjectInspector) oi).getPrimitiveWritableObject(o);
                binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes());
                break;
            }
        case INTERVAL_DAY_TIME:
            {
                wc = ((HiveIntervalDayTimeObjectInspector) oi).getPrimitiveWritableObject(o);
                binaryData = Base64.encodeBase64(String.valueOf(wc).getBytes());
                break;
            }
        case DECIMAL:
            {
                HiveDecimalObjectInspector decimalOI = (HiveDecimalObjectInspector) oi;
                binaryData = Base64.encodeBase64(String.valueOf(decimalOI).getBytes());
                break;
            }
        default:
            {
                throw new RuntimeException("Unknown primitive type: " + category);
            }
    }
    if (binaryData == null) {
        throw new RuntimeException("get primitive type is null: " + category);
    }
    out.write(binaryData, 0, binaryData.length);
}

Example 39 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project flink-mirror by flink-ci.

the class HiveInspectors method toFlinkObject.

/**
 * Converts a Hive object to Flink object with an ObjectInspector.
 */
public static Object toFlinkObject(ObjectInspector inspector, Object data, HiveShim hiveShim) {
    if (data == null || inspector instanceof VoidObjectInspector) {
        return null;
    }
    if (inspector instanceof PrimitiveObjectInspector) {
        if (inspector instanceof BooleanObjectInspector || inspector instanceof StringObjectInspector || inspector instanceof ByteObjectInspector || inspector instanceof ShortObjectInspector || inspector instanceof IntObjectInspector || inspector instanceof LongObjectInspector || inspector instanceof FloatObjectInspector || inspector instanceof DoubleObjectInspector || inspector instanceof BinaryObjectInspector) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
            return poi.getPrimitiveJavaObject(data);
        } else if (inspector instanceof DateObjectInspector) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
            return hiveShim.toFlinkDate(poi.getPrimitiveJavaObject(data));
        } else if (inspector instanceof TimestampObjectInspector) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
            return hiveShim.toFlinkTimestamp(poi.getPrimitiveJavaObject(data));
        } else if (inspector instanceof HiveCharObjectInspector) {
            HiveCharObjectInspector oi = (HiveCharObjectInspector) inspector;
            return oi.getPrimitiveJavaObject(data).getValue();
        } else if (inspector instanceof HiveVarcharObjectInspector) {
            HiveVarcharObjectInspector oi = (HiveVarcharObjectInspector) inspector;
            return oi.getPrimitiveJavaObject(data).getValue();
        } else if (inspector instanceof HiveDecimalObjectInspector) {
            HiveDecimalObjectInspector oi = (HiveDecimalObjectInspector) inspector;
            return oi.getPrimitiveJavaObject(data).bigDecimalValue();
        }
    }
    if (inspector instanceof ListObjectInspector) {
        ListObjectInspector listInspector = (ListObjectInspector) inspector;
        List<?> list = listInspector.getList(data);
        if (list == null) {
            return null;
        }
        // flink expects a specific array type (e.g. Integer[] instead of Object[]), so we have
        // to get the element class
        ObjectInspector elementInspector = listInspector.getListElementObjectInspector();
        Object[] result = (Object[]) Array.newInstance(HiveTypeUtil.toFlinkType(elementInspector).getConversionClass(), list.size());
        for (int i = 0; i < list.size(); i++) {
            result[i] = toFlinkObject(elementInspector, list.get(i), hiveShim);
        }
        return result;
    }
    if (inspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) inspector;
        Map<?, ?> map = mapInspector.getMap(data);
        if (map == null) {
            return null;
        }
        Map<Object, Object> result = new HashMap<>(map.size());
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            result.put(toFlinkObject(mapInspector.getMapKeyObjectInspector(), entry.getKey(), hiveShim), toFlinkObject(mapInspector.getMapValueObjectInspector(), entry.getValue(), hiveShim));
        }
        return result;
    }
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector structInspector = (StructObjectInspector) inspector;
        List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
        Row row = new Row(fields.size());
        // list as data
        if (!data.getClass().isArray() && !(data instanceof List) && (inspector instanceof StandardStructObjectInspector)) {
            data = new Object[] { data };
        }
        for (int i = 0; i < row.getArity(); i++) {
            row.setField(i, toFlinkObject(fields.get(i).getFieldObjectInspector(), structInspector.getStructFieldData(data, fields.get(i)), hiveShim));
        }
        return row;
    }
    throw new FlinkHiveUDFException(String.format("Unwrap does not support ObjectInspector '%s' yet", inspector));
}

Example 40 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project flink-mirror by flink-ci.

the class HiveInspectors method getConversion.

/**
 * Get conversion for converting Flink object to Hive object from an ObjectInspector and the
 * corresponding Flink DataType.
 */
public static HiveObjectConversion getConversion(ObjectInspector inspector, LogicalType dataType, HiveShim hiveShim) {
    if (inspector instanceof PrimitiveObjectInspector) {
        HiveObjectConversion conversion;
        if (inspector instanceof BooleanObjectInspector || inspector instanceof StringObjectInspector || inspector instanceof ByteObjectInspector || inspector instanceof ShortObjectInspector || inspector instanceof IntObjectInspector || inspector instanceof LongObjectInspector || inspector instanceof FloatObjectInspector || inspector instanceof DoubleObjectInspector || inspector instanceof BinaryObjectInspector || inspector instanceof VoidObjectInspector) {
            conversion = IdentityConversion.INSTANCE;
        } else if (inspector instanceof DateObjectInspector) {
            conversion = hiveShim::toHiveDate;
        } else if (inspector instanceof TimestampObjectInspector) {
            conversion = hiveShim::toHiveTimestamp;
        } else if (inspector instanceof HiveCharObjectInspector) {
            conversion = o -> o == null ? null : new HiveChar((String) o, ((CharType) dataType).getLength());
        } else if (inspector instanceof HiveVarcharObjectInspector) {
            conversion = o -> o == null ? null : new HiveVarchar((String) o, ((VarCharType) dataType).getLength());
        } else if (inspector instanceof HiveDecimalObjectInspector) {
            conversion = o -> o == null ? null : HiveDecimal.create((BigDecimal) o);
        } else {
            throw new FlinkHiveUDFException("Unsupported primitive object inspector " + inspector.getClass().getName());
        }
        // currently this happens for constant arguments for UDFs
        if (((PrimitiveObjectInspector) inspector).preferWritable()) {
            conversion = new WritableHiveObjectConversion(conversion, hiveShim);
        }
        return conversion;
    }
    if (inspector instanceof ListObjectInspector) {
        HiveObjectConversion eleConvert = getConversion(((ListObjectInspector) inspector).getListElementObjectInspector(), ((ArrayType) dataType).getElementType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Object[] array = (Object[]) o;
            List<Object> result = new ArrayList<>();
            for (Object ele : array) {
                result.add(eleConvert.toHiveObject(ele));
            }
            return result;
        };
    }
    if (inspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) inspector;
        MapType kvType = (MapType) dataType;
        HiveObjectConversion keyConversion = getConversion(mapInspector.getMapKeyObjectInspector(), kvType.getKeyType(), hiveShim);
        HiveObjectConversion valueConversion = getConversion(mapInspector.getMapValueObjectInspector(), kvType.getValueType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Map<Object, Object> map = (Map) o;
            Map<Object, Object> result = new HashMap<>(map.size());
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                result.put(keyConversion.toHiveObject(entry.getKey()), valueConversion.toHiveObject(entry.getValue()));
            }
            return result;
        };
    }
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector structInspector = (StructObjectInspector) inspector;
        List<? extends StructField> structFields = structInspector.getAllStructFieldRefs();
        List<RowType.RowField> rowFields = ((RowType) dataType).getFields();
        HiveObjectConversion[] conversions = new HiveObjectConversion[structFields.size()];
        for (int i = 0; i < structFields.size(); i++) {
            conversions[i] = getConversion(structFields.get(i).getFieldObjectInspector(), rowFields.get(i).getType(), hiveShim);
        }
        return o -> {
            if (o == null) {
                return null;
            }
            Row row = (Row) o;
            List<Object> result = new ArrayList<>(row.getArity());
            for (int i = 0; i < row.getArity(); i++) {
                result.add(conversions[i].toHiveObject(row.getField(i)));
            }
            return result;
        };
    }
    throw new FlinkHiveUDFException(String.format("Flink doesn't support convert object conversion for %s yet", inspector));
}