Examples with IntObjectInspector org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector used on opensource projects

Example 21 with IntObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector in project flink by apache.

the class HiveInspectors method getConversion.

/**
 * Get conversion for converting Flink object to Hive object from an ObjectInspector and the
 * corresponding Flink DataType.
 */
public static HiveObjectConversion getConversion(ObjectInspector inspector, LogicalType dataType, HiveShim hiveShim) {
    if (inspector instanceof PrimitiveObjectInspector) {
        HiveObjectConversion conversion;
        if (inspector instanceof BooleanObjectInspector || inspector instanceof StringObjectInspector || inspector instanceof ByteObjectInspector || inspector instanceof ShortObjectInspector || inspector instanceof IntObjectInspector || inspector instanceof LongObjectInspector || inspector instanceof FloatObjectInspector || inspector instanceof DoubleObjectInspector || inspector instanceof BinaryObjectInspector || inspector instanceof VoidObjectInspector) {
            conversion = IdentityConversion.INSTANCE;
        } else if (inspector instanceof DateObjectInspector) {
            conversion = hiveShim::toHiveDate;
        } else if (inspector instanceof TimestampObjectInspector) {
            conversion = hiveShim::toHiveTimestamp;
        } else if (inspector instanceof HiveCharObjectInspector) {
            conversion = o -> o == null ? null : new HiveChar((String) o, ((CharType) dataType).getLength());
        } else if (inspector instanceof HiveVarcharObjectInspector) {
            conversion = o -> o == null ? null : new HiveVarchar((String) o, ((VarCharType) dataType).getLength());
        } else if (inspector instanceof HiveDecimalObjectInspector) {
            conversion = o -> o == null ? null : HiveDecimal.create((BigDecimal) o);
        } else {
            throw new FlinkHiveUDFException("Unsupported primitive object inspector " + inspector.getClass().getName());
        }
        // currently this happens for constant arguments for UDFs
        if (((PrimitiveObjectInspector) inspector).preferWritable()) {
            conversion = new WritableHiveObjectConversion(conversion, hiveShim);
        }
        return conversion;
    }
    if (inspector instanceof ListObjectInspector) {
        HiveObjectConversion eleConvert = getConversion(((ListObjectInspector) inspector).getListElementObjectInspector(), ((ArrayType) dataType).getElementType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Object[] array = (Object[]) o;
            List<Object> result = new ArrayList<>();
            for (Object ele : array) {
                result.add(eleConvert.toHiveObject(ele));
            }
            return result;
        };
    }
    if (inspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) inspector;
        MapType kvType = (MapType) dataType;
        HiveObjectConversion keyConversion = getConversion(mapInspector.getMapKeyObjectInspector(), kvType.getKeyType(), hiveShim);
        HiveObjectConversion valueConversion = getConversion(mapInspector.getMapValueObjectInspector(), kvType.getValueType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Map<Object, Object> map = (Map) o;
            Map<Object, Object> result = new HashMap<>(map.size());
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                result.put(keyConversion.toHiveObject(entry.getKey()), valueConversion.toHiveObject(entry.getValue()));
            }
            return result;
        };
    }
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector structInspector = (StructObjectInspector) inspector;
        List<? extends StructField> structFields = structInspector.getAllStructFieldRefs();
        List<RowType.RowField> rowFields = ((RowType) dataType).getFields();
        HiveObjectConversion[] conversions = new HiveObjectConversion[structFields.size()];
        for (int i = 0; i < structFields.size(); i++) {
            conversions[i] = getConversion(structFields.get(i).getFieldObjectInspector(), rowFields.get(i).getType(), hiveShim);
        }
        return o -> {
            if (o == null) {
                return null;
            }
            Row row = (Row) o;
            List<Object> result = new ArrayList<>(row.getArity());
            for (int i = 0; i < row.getArity(); i++) {
                result.add(conversions[i].toHiveObject(row.getField(i)));
            }
            return result;
        };
    }
    throw new FlinkHiveUDFException(String.format("Flink doesn't support convert object conversion for %s yet", inspector));
}

Example 22 with IntObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector in project flink by apache.

the class HiveInspectors method toFlinkObject.

/**
 * Converts a Hive object to Flink object with an ObjectInspector.
 */
public static Object toFlinkObject(ObjectInspector inspector, Object data, HiveShim hiveShim) {
    if (data == null || inspector instanceof VoidObjectInspector) {
        return null;
    }
    if (inspector instanceof PrimitiveObjectInspector) {
        if (inspector instanceof BooleanObjectInspector || inspector instanceof StringObjectInspector || inspector instanceof ByteObjectInspector || inspector instanceof ShortObjectInspector || inspector instanceof IntObjectInspector || inspector instanceof LongObjectInspector || inspector instanceof FloatObjectInspector || inspector instanceof DoubleObjectInspector || inspector instanceof BinaryObjectInspector) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
            return poi.getPrimitiveJavaObject(data);
        } else if (inspector instanceof DateObjectInspector) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
            return hiveShim.toFlinkDate(poi.getPrimitiveJavaObject(data));
        } else if (inspector instanceof TimestampObjectInspector) {
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) inspector;
            return hiveShim.toFlinkTimestamp(poi.getPrimitiveJavaObject(data));
        } else if (inspector instanceof HiveCharObjectInspector) {
            HiveCharObjectInspector oi = (HiveCharObjectInspector) inspector;
            return oi.getPrimitiveJavaObject(data).getValue();
        } else if (inspector instanceof HiveVarcharObjectInspector) {
            HiveVarcharObjectInspector oi = (HiveVarcharObjectInspector) inspector;
            return oi.getPrimitiveJavaObject(data).getValue();
        } else if (inspector instanceof HiveDecimalObjectInspector) {
            HiveDecimalObjectInspector oi = (HiveDecimalObjectInspector) inspector;
            return oi.getPrimitiveJavaObject(data).bigDecimalValue();
        }
    }
    if (inspector instanceof ListObjectInspector) {
        ListObjectInspector listInspector = (ListObjectInspector) inspector;
        List<?> list = listInspector.getList(data);
        if (list == null) {
            return null;
        }
        // flink expects a specific array type (e.g. Integer[] instead of Object[]), so we have
        // to get the element class
        ObjectInspector elementInspector = listInspector.getListElementObjectInspector();
        Object[] result = (Object[]) Array.newInstance(HiveTypeUtil.toFlinkType(elementInspector).getConversionClass(), list.size());
        for (int i = 0; i < list.size(); i++) {
            result[i] = toFlinkObject(elementInspector, list.get(i), hiveShim);
        }
        return result;
    }
    if (inspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) inspector;
        Map<?, ?> map = mapInspector.getMap(data);
        if (map == null) {
            return null;
        }
        Map<Object, Object> result = new HashMap<>(map.size());
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            result.put(toFlinkObject(mapInspector.getMapKeyObjectInspector(), entry.getKey(), hiveShim), toFlinkObject(mapInspector.getMapValueObjectInspector(), entry.getValue(), hiveShim));
        }
        return result;
    }
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector structInspector = (StructObjectInspector) inspector;
        List<? extends StructField> fields = structInspector.getAllStructFieldRefs();
        Row row = new Row(fields.size());
        // list as data
        if (!data.getClass().isArray() && !(data instanceof List) && (inspector instanceof StandardStructObjectInspector)) {
            data = new Object[] { data };
        }
        for (int i = 0; i < row.getArity(); i++) {
            row.setField(i, toFlinkObject(fields.get(i).getFieldObjectInspector(), structInspector.getStructFieldData(data, fields.get(i)), hiveShim));
        }
        return row;
    }
    throw new FlinkHiveUDFException(String.format("Unwrap does not support ObjectInspector '%s' yet", inspector));
}

Example 23 with IntObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector in project hive by apache.

the class TypedBytesSerDe method serializeField.

private void serializeField(Object o, ObjectInspector oi, Object reuse) throws IOException {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        {
                            return;
                        }
                    case BOOLEAN:
                        {
                            BooleanObjectInspector boi = (BooleanObjectInspector) poi;
                            BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse;
                            r.set(boi.get(o));
                            tbOut.write(r);
                            return;
                        }
                    case BYTE:
                        {
                            ByteObjectInspector boi = (ByteObjectInspector) poi;
                            ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse;
                            r.set(boi.get(o));
                            tbOut.write(r);
                            return;
                        }
                    case SHORT:
                        {
                            ShortObjectInspector spoi = (ShortObjectInspector) poi;
                            ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse;
                            r.set(spoi.get(o));
                            tbOut.write(r);
                            return;
                        }
                    case INT:
                        {
                            IntObjectInspector ioi = (IntObjectInspector) poi;
                            IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse;
                            r.set(ioi.get(o));
                            tbOut.write(r);
                            return;
                        }
                    case LONG:
                        {
                            LongObjectInspector loi = (LongObjectInspector) poi;
                            LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse;
                            r.set(loi.get(o));
                            tbOut.write(r);
                            return;
                        }
                    case FLOAT:
                        {
                            FloatObjectInspector foi = (FloatObjectInspector) poi;
                            FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse;
                            r.set(foi.get(o));
                            tbOut.write(r);
                            return;
                        }
                    case DOUBLE:
                        {
                            DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                            DoubleWritable r = reuse == null ? new DoubleWritable() : (DoubleWritable) reuse;
                            r.set(doi.get(o));
                            tbOut.write(r);
                            return;
                        }
                    case STRING:
                        {
                            StringObjectInspector soi = (StringObjectInspector) poi;
                            Text t = soi.getPrimitiveWritableObject(o);
                            tbOut.write(t);
                            return;
                        }
                    default:
                        {
                            throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
        case MAP:
        case STRUCT:
            {
                // For complex object, serialize to JSON format
                String s = SerDeUtils.getJSONString(o, oi);
                Text t = reuse == null ? new Text() : (Text) reuse;
                // convert to Text and write it
                t.set(s);
                tbOut.write(t);
            }
        default:
            {
                throw new RuntimeException("Unrecognized type: " + oi.getCategory());
            }
    }
}

Example 24 with IntObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector in project hive by apache.

the class DruidSerDe method serialize.

@Override
public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException {
    if (objectInspector.getCategory() != ObjectInspector.Category.STRUCT) {
        throw new SerDeException(getClass().toString() + " can only serialize struct types, but we got: " + objectInspector.getTypeName());
    }
    // Prepare the field ObjectInspectors
    StructObjectInspector soi = (StructObjectInspector) objectInspector;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();
    List<Object> values = soi.getStructFieldsDataAsList(o);
    // We deserialize the result
    final Map<String, Object> value = new HashMap<>();
    for (int i = 0; i < columns.length; i++) {
        if (values.get(i) == null) {
            // null, we just add it
            value.put(columns[i], null);
            continue;
        }
        final Object res;
        switch(types[i].getPrimitiveCategory()) {
            case TIMESTAMP:
                res = ((TimestampObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)).toEpochMilli();
                break;
            case TIMESTAMPLOCALTZ:
                res = ((TimestampLocalTZObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)).getZonedDateTime().toInstant().toEpochMilli();
                break;
            case BYTE:
                res = ((ByteObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
                break;
            case SHORT:
                res = ((ShortObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
                break;
            case INT:
                res = ((IntObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
                break;
            case LONG:
                res = ((LongObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
                break;
            case FLOAT:
                res = ((FloatObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
                break;
            case DOUBLE:
                res = ((DoubleObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
                break;
            case CHAR:
                res = ((HiveCharObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)).getValue();
                break;
            case VARCHAR:
                res = ((HiveVarcharObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i)).getValue();
                break;
            case STRING:
                res = ((StringObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i));
                break;
            case BOOLEAN:
                res = ((BooleanObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)) ? 1L : 0L;
                break;
            default:
                throw new SerDeException("Unsupported type: " + types[i].getPrimitiveCategory());
        }
        value.put(columns[i], res);
    }
    // Extract the partitions keys segments granularity and partition key if any
    // First Segment Granularity has to be here.
    final int granularityFieldIndex = columns.length;
    assert values.size() > granularityFieldIndex;
    Preconditions.checkArgument(fields.get(granularityFieldIndex).getFieldName().equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME));
    Timestamp timestamp = ((TimestampObjectInspector) fields.get(granularityFieldIndex).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(granularityFieldIndex));
    Preconditions.checkNotNull(timestamp, "Timestamp column cannot have null value");
    value.put(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, timestamp.toEpochMilli());
    if (values.size() == columns.length + 2) {
        // Then partition number if any.
        final int partitionNumPos = granularityFieldIndex + 1;
        Preconditions.checkArgument(fields.get(partitionNumPos).getFieldName().equals(Constants.DRUID_SHARD_KEY_COL_NAME), String.format("expecting to encounter %s but was %s", Constants.DRUID_SHARD_KEY_COL_NAME, fields.get(partitionNumPos).getFieldName()));
        value.put(Constants.DRUID_SHARD_KEY_COL_NAME, ((LongObjectInspector) fields.get(partitionNumPos).getFieldObjectInspector()).get(values.get(partitionNumPos)));
    }
    return new DruidWritable(value);
}

Example 25 with IntObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector in project hive by apache.

the class ObjectInspectorUtils method hashCode.

public static int hashCode(Object o, ObjectInspector objIns) {
    if (o == null) {
        return 0;
    }
    switch(objIns.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns);
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        return 0;
                    case BOOLEAN:
                        return ((BooleanObjectInspector) poi).get(o) ? 1 : 0;
                    case BYTE:
                        return ((ByteObjectInspector) poi).get(o);
                    case SHORT:
                        return ((ShortObjectInspector) poi).get(o);
                    case INT:
                        return ((IntObjectInspector) poi).get(o);
                    case LONG:
                        {
                            long a = ((LongObjectInspector) poi).get(o);
                            return (int) ((a >>> 32) ^ a);
                        }
                    case FLOAT:
                        return Float.floatToIntBits(((FloatObjectInspector) poi).get(o));
                    case DOUBLE:
                        {
                            // This hash function returns the same result as Double.hashCode()
                            // while DoubleWritable.hashCode returns a different result.
                            long a = Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o));
                            return (int) ((a >>> 32) ^ a);
                        }
                    case STRING:
                        {
                            // This hash function returns the same result as String.hashCode() when
                            // all characters are ASCII, while Text.hashCode() always returns a
                            // different result.
                            Text t = ((StringObjectInspector) poi).getPrimitiveWritableObject(o);
                            int r = 0;
                            for (int i = 0; i < t.getLength(); i++) {
                                r = r * 31 + t.getBytes()[i];
                            }
                            return r;
                        }
                    case CHAR:
                        return ((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case VARCHAR:
                        return ((HiveVarcharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case BINARY:
                        return ((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case DATE:
                        return ((DateObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case TIMESTAMP:
                        // Use old timestamp writable hash code for backwards compatibility
                        TimestampWritable ts = new TimestampWritable(java.sql.Timestamp.valueOf(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o).toString()));
                        return ts.hashCode();
                    case TIMESTAMPLOCALTZ:
                        TimestampLocalTZWritable tstz = ((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o);
                        return tstz.hashCode();
                    case INTERVAL_YEAR_MONTH:
                        HiveIntervalYearMonthWritable intervalYearMonth = ((HiveIntervalYearMonthObjectInspector) poi).getPrimitiveWritableObject(o);
                        return intervalYearMonth.hashCode();
                    case INTERVAL_DAY_TIME:
                        HiveIntervalDayTimeWritable intervalDayTime = ((HiveIntervalDayTimeObjectInspector) poi).getPrimitiveWritableObject(o);
                        return intervalDayTime.hashCode();
                    case DECIMAL:
                        // compatible hash code.
                        return ((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    default:
                        {
                            throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                int r = 0;
                ListObjectInspector listOI = (ListObjectInspector) objIns;
                ObjectInspector elemOI = listOI.getListElementObjectInspector();
                for (int ii = 0; ii < listOI.getListLength(o); ++ii) {
                    r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI);
                }
                return r;
            }
        case MAP:
            {
                int r = 0;
                MapObjectInspector mapOI = (MapObjectInspector) objIns;
                ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
                ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
                Map<?, ?> map = mapOI.getMap(o);
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI);
                }
                return r;
            }
        case STRUCT:
            int r = 0;
            StructObjectInspector structOI = (StructObjectInspector) objIns;
            List<? extends StructField> fields = structOI.getAllStructFieldRefs();
            for (StructField field : fields) {
                r = 31 * r + hashCode(structOI.getStructFieldData(o, field), field.getFieldObjectInspector());
            }
            return r;
        case UNION:
            UnionObjectInspector uOI = (UnionObjectInspector) objIns;
            byte tag = uOI.getTag(o);
            return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
        default:
            throw new RuntimeException("Unknown type: " + objIns.getTypeName());
    }
}