Examples with DateObjectInspector org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector used on opensource projects

Example 26 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project hive by apache.

the class ColumnStatisticsObjTranslator method unpackDateStats.

private static void unpackDateStats(ObjectInspector oi, Object o, ColumnStatsField csf, ColumnStatisticsObj statsObj) {
    switch(csf) {
        case COUNT_NULLS:
            long cn = ((LongObjectInspector) oi).get(o);
            statsObj.getStatsData().getDateStats().setNumNulls(cn);
            break;
        case MIN:
            DateWritableV2 min = ((DateObjectInspector) oi).getPrimitiveWritableObject(o);
            statsObj.getStatsData().getDateStats().setLowValue(new Date(min.getDays()));
            break;
        case MAX:
            DateWritableV2 max = ((DateObjectInspector) oi).getPrimitiveWritableObject(o);
            statsObj.getStatsData().getDateStats().setHighValue(new Date(max.getDays()));
            break;
        case NDV:
            long ndv = ((LongObjectInspector) oi).get(o);
            statsObj.getStatsData().getDateStats().setNumDVs(ndv);
            break;
        case BITVECTOR:
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o);
            statsObj.getStatsData().getDateStats().setBitVectors(buf);
            break;
        default:
            throw new RuntimeException("Unsupported column stat for DATE : " + csf);
    }
}

Example 27 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project hive by apache.

the class GenericUDFCastFormat method convert.

private Object convert(Object o) throws HiveException {
    Object input;
    switch(inputOI.getPrimitiveCategory()) {
        case STRING:
            input = ((StringObjectInspector) inputOI).getPrimitiveJavaObject(o);
            break;
        case CHAR:
            input = ((HiveCharObjectInspector) inputOI).getPrimitiveJavaObject(o).getStrippedValue();
            break;
        case VARCHAR:
            input = ((HiveVarcharObjectInspector) inputOI).getPrimitiveJavaObject(o).toString();
            break;
        case TIMESTAMP:
            input = ((TimestampObjectInspector) inputOI).getPrimitiveWritableObject(o).getTimestamp();
            break;
        case DATE:
            input = ((DateObjectInspector) inputOI).getPrimitiveWritableObject(o).get();
            break;
        default:
            throw new HiveException("Input type " + inputOI.getPrimitiveCategory() + " not valid");
    }
    // format here
    Object formattedOutput = null;
    if (inputOI.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.DATE || inputOI.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP) {
        if (inputOI.getPrimitiveCategory() == PrimitiveObjectInspector.PrimitiveCategory.DATE) {
            try {
                formattedOutput = formatter.format((Date) input);
            } catch (IllegalArgumentException e) {
                return null;
            }
        } else {
            try {
                formattedOutput = formatter.format((Timestamp) input);
            } catch (IllegalArgumentException e) {
                return null;
            }
        }
        if (formattedOutput == null) {
            return null;
        }
    }
    // parse and create Writables
    switch(outputOI.getPrimitiveCategory()) {
        case STRING:
            return new Text((String) formattedOutput);
        case CHAR:
            return ((SettableHiveCharObjectInspector) outputOI).create(new HiveChar((String) formattedOutput, -1));
        case VARCHAR:
            return ((SettableHiveVarcharObjectInspector) outputOI).create(new HiveVarchar((String) formattedOutput, -1));
        case TIMESTAMP:
            try {
                Timestamp t = formatter.parseTimestamp((String) input);
                if (t == null) {
                    return null;
                }
                return ((SettableTimestampObjectInspector) outputOI).create(t);
            } catch (IllegalArgumentException e) {
                return null;
            }
        case DATE:
            try {
                Date d = formatter.parseDate((String) input);
                if (d == null) {
                    return null;
                }
                return ((SettableDateObjectInspector) outputOI).create(d);
            } catch (IllegalArgumentException e) {
                return null;
            }
        default:
            throw new HiveException("Output type " + outputOI.getPrimitiveCategory() + " not valid");
    }
}

Example 28 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project hive by apache.

the class ObjectInspectorUtils method hashCode.

public static int hashCode(Object o, ObjectInspector objIns) {
    if (o == null) {
        return 0;
    }
    switch(objIns.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns);
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        return 0;
                    case BOOLEAN:
                        return ((BooleanObjectInspector) poi).get(o) ? 1 : 0;
                    case BYTE:
                        return ((ByteObjectInspector) poi).get(o);
                    case SHORT:
                        return ((ShortObjectInspector) poi).get(o);
                    case INT:
                        return ((IntObjectInspector) poi).get(o);
                    case LONG:
                        {
                            long a = ((LongObjectInspector) poi).get(o);
                            return (int) ((a >>> 32) ^ a);
                        }
                    case FLOAT:
                        return Float.floatToIntBits(((FloatObjectInspector) poi).get(o));
                    case DOUBLE:
                        {
                            // This hash function returns the same result as Double.hashCode()
                            // while DoubleWritable.hashCode returns a different result.
                            long a = Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o));
                            return (int) ((a >>> 32) ^ a);
                        }
                    case STRING:
                        {
                            // This hash function returns the same result as String.hashCode() when
                            // all characters are ASCII, while Text.hashCode() always returns a
                            // different result.
                            Text t = ((StringObjectInspector) poi).getPrimitiveWritableObject(o);
                            int r = 0;
                            for (int i = 0; i < t.getLength(); i++) {
                                r = r * 31 + t.getBytes()[i];
                            }
                            return r;
                        }
                    case CHAR:
                        return ((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case VARCHAR:
                        return ((HiveVarcharObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case BINARY:
                        return ((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case DATE:
                        return ((DateObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    case TIMESTAMP:
                        // Use old timestamp writable hash code for backwards compatibility
                        TimestampWritable ts = new TimestampWritable(java.sql.Timestamp.valueOf(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o).toString()));
                        return ts.hashCode();
                    case TIMESTAMPLOCALTZ:
                        TimestampLocalTZWritable tstz = ((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o);
                        return tstz.hashCode();
                    case INTERVAL_YEAR_MONTH:
                        HiveIntervalYearMonthWritable intervalYearMonth = ((HiveIntervalYearMonthObjectInspector) poi).getPrimitiveWritableObject(o);
                        return intervalYearMonth.hashCode();
                    case INTERVAL_DAY_TIME:
                        HiveIntervalDayTimeWritable intervalDayTime = ((HiveIntervalDayTimeObjectInspector) poi).getPrimitiveWritableObject(o);
                        return intervalDayTime.hashCode();
                    case DECIMAL:
                        // compatible hash code.
                        return ((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).hashCode();
                    default:
                        {
                            throw new RuntimeException("Unknown type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                int r = 0;
                ListObjectInspector listOI = (ListObjectInspector) objIns;
                ObjectInspector elemOI = listOI.getListElementObjectInspector();
                for (int ii = 0; ii < listOI.getListLength(o); ++ii) {
                    r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI);
                }
                return r;
            }
        case MAP:
            {
                int r = 0;
                MapObjectInspector mapOI = (MapObjectInspector) objIns;
                ObjectInspector keyOI = mapOI.getMapKeyObjectInspector();
                ObjectInspector valueOI = mapOI.getMapValueObjectInspector();
                Map<?, ?> map = mapOI.getMap(o);
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    r += hashCode(entry.getKey(), keyOI) ^ hashCode(entry.getValue(), valueOI);
                }
                return r;
            }
        case STRUCT:
            int r = 0;
            StructObjectInspector structOI = (StructObjectInspector) objIns;
            List<? extends StructField> fields = structOI.getAllStructFieldRefs();
            for (StructField field : fields) {
                r = 31 * r + hashCode(structOI.getStructFieldData(o, field), field.getFieldObjectInspector());
            }
            return r;
        case UNION:
            UnionObjectInspector uOI = (UnionObjectInspector) objIns;
            byte tag = uOI.getTag(o);
            return hashCode(uOI.getField(o), uOI.getObjectInspectors().get(tag));
        default:
            throw new RuntimeException("Unknown type: " + objIns.getTypeName());
    }
}

Example 29 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project hive by apache.

the class BinarySortableSerDe method serialize.

static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, boolean invert, byte nullMarker, byte notNullMarker) throws SerDeException {
    // Is this field a null?
    if (o == null) {
        writeByte(buffer, nullMarker, invert);
        return;
    }
    // This field is not a null.
    writeByte(buffer, notNullMarker, invert);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
                switch(poi.getPrimitiveCategory()) {
                    case VOID:
                        {
                            return;
                        }
                    case BOOLEAN:
                        {
                            boolean v = ((BooleanObjectInspector) poi).get(o);
                            writeByte(buffer, (byte) (v ? 2 : 1), invert);
                            return;
                        }
                    case BYTE:
                        {
                            ByteObjectInspector boi = (ByteObjectInspector) poi;
                            byte v = boi.get(o);
                            writeByte(buffer, (byte) (v ^ 0x80), invert);
                            return;
                        }
                    case SHORT:
                        {
                            ShortObjectInspector spoi = (ShortObjectInspector) poi;
                            short v = spoi.get(o);
                            serializeShort(buffer, v, invert);
                            return;
                        }
                    case INT:
                        {
                            IntObjectInspector ioi = (IntObjectInspector) poi;
                            int v = ioi.get(o);
                            serializeInt(buffer, v, invert);
                            return;
                        }
                    case LONG:
                        {
                            LongObjectInspector loi = (LongObjectInspector) poi;
                            long v = loi.get(o);
                            serializeLong(buffer, v, invert);
                            return;
                        }
                    case FLOAT:
                        {
                            FloatObjectInspector foi = (FloatObjectInspector) poi;
                            serializeFloat(buffer, foi.get(o), invert);
                            return;
                        }
                    case DOUBLE:
                        {
                            DoubleObjectInspector doi = (DoubleObjectInspector) poi;
                            serializeDouble(buffer, doi.get(o), invert);
                            return;
                        }
                    case STRING:
                        {
                            StringObjectInspector soi = (StringObjectInspector) poi;
                            Text t = soi.getPrimitiveWritableObject(o);
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case CHAR:
                        {
                            HiveCharObjectInspector hcoi = (HiveCharObjectInspector) poi;
                            HiveCharWritable hc = hcoi.getPrimitiveWritableObject(o);
                            // Trailing space should ignored for char comparisons.
                            // So write stripped values for this SerDe.
                            Text t = hc.getStrippedValue();
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case VARCHAR:
                        {
                            HiveVarcharObjectInspector hcoi = (HiveVarcharObjectInspector) poi;
                            HiveVarcharWritable hc = hcoi.getPrimitiveWritableObject(o);
                            // use varchar's text field directly
                            Text t = hc.getTextValue();
                            serializeBytes(buffer, t.getBytes(), t.getLength(), invert);
                            return;
                        }
                    case BINARY:
                        {
                            BinaryObjectInspector baoi = (BinaryObjectInspector) poi;
                            BytesWritable ba = baoi.getPrimitiveWritableObject(o);
                            byte[] toSer = new byte[ba.getLength()];
                            System.arraycopy(ba.getBytes(), 0, toSer, 0, ba.getLength());
                            serializeBytes(buffer, toSer, ba.getLength(), invert);
                            return;
                        }
                    case DATE:
                        {
                            DateObjectInspector doi = (DateObjectInspector) poi;
                            int v = doi.getPrimitiveWritableObject(o).getDays();
                            serializeInt(buffer, v, invert);
                            return;
                        }
                    case TIMESTAMP:
                        {
                            TimestampObjectInspector toi = (TimestampObjectInspector) poi;
                            TimestampWritableV2 t = toi.getPrimitiveWritableObject(o);
                            serializeTimestampWritable(buffer, t, invert);
                            return;
                        }
                    case TIMESTAMPLOCALTZ:
                        {
                            TimestampLocalTZObjectInspector toi = (TimestampLocalTZObjectInspector) poi;
                            TimestampLocalTZWritable t = toi.getPrimitiveWritableObject(o);
                            serializeTimestampTZWritable(buffer, t, invert);
                            return;
                        }
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonthObjectInspector ioi = (HiveIntervalYearMonthObjectInspector) poi;
                            HiveIntervalYearMonth intervalYearMonth = ioi.getPrimitiveJavaObject(o);
                            serializeHiveIntervalYearMonth(buffer, intervalYearMonth, invert);
                            return;
                        }
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTimeObjectInspector ioi = (HiveIntervalDayTimeObjectInspector) poi;
                            HiveIntervalDayTime intervalDayTime = ioi.getPrimitiveJavaObject(o);
                            serializeHiveIntervalDayTime(buffer, intervalDayTime, invert);
                            return;
                        }
                    case DECIMAL:
                        {
                            HiveDecimalObjectInspector boi = (HiveDecimalObjectInspector) poi;
                            HiveDecimal dec = boi.getPrimitiveJavaObject(o);
                            serializeHiveDecimal(buffer, dec, invert);
                            return;
                        }
                    default:
                        {
                            throw new RuntimeException("Unrecognized type: " + poi.getPrimitiveCategory());
                        }
                }
            }
        case LIST:
            {
                ListObjectInspector loi = (ListObjectInspector) oi;
                ObjectInspector eoi = loi.getListElementObjectInspector();
                // \1 followed by each element
                int size = loi.getListLength(o);
                for (int eid = 0; eid < size; eid++) {
                    writeByte(buffer, (byte) 1, invert);
                    serialize(buffer, loi.getListElement(o, eid), eoi, invert, nullMarker, notNullMarker);
                }
                // and \0 to terminate
                writeByte(buffer, (byte) 0, invert);
                return;
            }
        case MAP:
            {
                MapObjectInspector moi = (MapObjectInspector) oi;
                ObjectInspector koi = moi.getMapKeyObjectInspector();
                ObjectInspector voi = moi.getMapValueObjectInspector();
                // \1 followed by each key and then each value
                Map<?, ?> map = moi.getMap(o);
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    writeByte(buffer, (byte) 1, invert);
                    serialize(buffer, entry.getKey(), koi, invert, nullMarker, notNullMarker);
                    serialize(buffer, entry.getValue(), voi, invert, nullMarker, notNullMarker);
                }
                // and \0 to terminate
                writeByte(buffer, (byte) 0, invert);
                return;
            }
        case STRUCT:
            {
                StructObjectInspector soi = (StructObjectInspector) oi;
                List<? extends StructField> fields = soi.getAllStructFieldRefs();
                for (int i = 0; i < fields.size(); i++) {
                    serialize(buffer, soi.getStructFieldData(o, fields.get(i)), fields.get(i).getFieldObjectInspector(), invert, nullMarker, notNullMarker);
                }
                return;
            }
        case UNION:
            {
                UnionObjectInspector uoi = (UnionObjectInspector) oi;
                byte tag = uoi.getTag(o);
                writeByte(buffer, tag, invert);
                serialize(buffer, uoi.getField(o), uoi.getObjectInspectors().get(tag), invert, nullMarker, notNullMarker);
                return;
            }
        default:
            {
                throw new RuntimeException("Unrecognized type: " + oi.getCategory());
            }
    }
}

Example 30 with DateObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector in project hive by apache.

the class AvroSerializer method serializePrimitive.

private Object serializePrimitive(TypeInfo typeInfo, PrimitiveObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException {
    switch(fieldOI.getPrimitiveCategory()) {
        case BINARY:
            if (schema.getType() == Type.BYTES) {
                return AvroSerdeUtils.getBufferFromBytes((byte[]) fieldOI.getPrimitiveJavaObject(structFieldData));
            } else if (schema.getType() == Type.FIXED) {
                Fixed fixed = new GenericData.Fixed(schema, (byte[]) fieldOI.getPrimitiveJavaObject(structFieldData));
                return fixed;
            } else {
                throw new AvroSerdeException("Unexpected Avro schema for Binary TypeInfo: " + schema.getType());
            }
        case DECIMAL:
            HiveDecimal dec = (HiveDecimal) fieldOI.getPrimitiveJavaObject(structFieldData);
            return AvroSerdeUtils.getBufferFromDecimal(dec, ((DecimalTypeInfo) typeInfo).scale());
        case CHAR:
            HiveChar ch = (HiveChar) fieldOI.getPrimitiveJavaObject(structFieldData);
            return ch.getStrippedValue();
        case VARCHAR:
            HiveVarchar vc = (HiveVarchar) fieldOI.getPrimitiveJavaObject(structFieldData);
            return vc.getValue();
        case DATE:
            Date date = ((DateObjectInspector) fieldOI).getPrimitiveJavaObject(structFieldData);
            return defaultProleptic ? date.toEpochDay() : CalendarUtils.convertDateToHybrid(date.toEpochDay());
        case TIMESTAMP:
            Timestamp timestamp = ((TimestampObjectInspector) fieldOI).getPrimitiveJavaObject(structFieldData);
            long millis = defaultProleptic ? timestamp.toEpochMilli() : CalendarUtils.convertTimeToHybrid(timestamp.toEpochMilli());
            timestamp = TimestampTZUtil.convertTimestampToZone(Timestamp.ofEpochMilli(millis), TimeZone.getDefault().toZoneId(), ZoneOffset.UTC, legacyConversion);
            return timestamp.toEpochMilli();
        case UNKNOWN:
            throw new AvroSerdeException("Received UNKNOWN primitive category.");
        case VOID:
            return null;
        default:
            // All other primitive types are simple
            return fieldOI.getPrimitiveJavaObject(structFieldData);
    }
}