Examples with DateWritableV2 org.apache.hadoop.hive.serde2.io.DateWritableV2 used on opensource projects

Example 56 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class AvroDeserializer method deserializePrimitive.

private Object deserializePrimitive(Object datum, Schema fileSchema, Schema recordSchema, PrimitiveTypeInfo columnType) throws AvroSerdeException {
    switch(columnType.getPrimitiveCategory()) {
        case STRING:
            // To workaround AvroUTF8
            return datum.toString();
        // and convert it to a string. Yay!
        case BINARY:
            if (recordSchema.getType() == Type.FIXED) {
                Fixed fixed = (Fixed) datum;
                return fixed.bytes();
            } else if (recordSchema.getType() == Type.BYTES) {
                return AvroSerdeUtils.getBytesFromByteBuffer((ByteBuffer) datum);
            } else {
                throw new AvroSerdeException("Unexpected Avro schema for Binary TypeInfo: " + recordSchema.getType());
            }
        case DECIMAL:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for decimal field. Reader schema is " + columnType);
            }
            int scale = 0;
            try {
                scale = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_SCALE);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex);
            }
            HiveDecimal dec = AvroSerdeUtils.getHiveDecimalFromByteBuffer((ByteBuffer) datum, scale);
            JavaHiveDecimalObjectInspector oi = (JavaHiveDecimalObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector((DecimalTypeInfo) columnType);
            return oi.set(null, dec);
        case CHAR:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for char field. Reader schema is " + columnType);
            }
            int maxLength = 0;
            try {
                maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain maxLength value for char field from file schema: " + fileSchema, ex);
            }
            String str = datum.toString();
            HiveChar hc = new HiveChar(str, maxLength);
            return hc;
        case VARCHAR:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for varchar field. Reader schema is " + columnType);
            }
            maxLength = 0;
            try {
                maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain maxLength value for varchar field from file schema: " + fileSchema, ex);
            }
            str = datum.toString();
            HiveVarchar hvc = new HiveVarchar(str, maxLength);
            return hvc;
        case DATE:
            {
                if (recordSchema.getType() != Type.INT) {
                    throw new AvroSerdeException("Unexpected Avro schema for Date TypeInfo: " + recordSchema.getType());
                }
                final boolean skipProlepticConversion;
                if (writerProleptic != null) {
                    skipProlepticConversion = writerProleptic;
                } else {
                    if (configuration != null) {
                        skipProlepticConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT);
                    } else {
                        skipProlepticConversion = HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.defaultBoolVal;
                    }
                }
                return Date.ofEpochMilli(DateWritableV2.daysToMillis(skipProlepticConversion ? (Integer) datum : CalendarUtils.convertDateToProleptic((Integer) datum)));
            }
        case TIMESTAMP:
            {
                if (recordSchema.getType() != Type.LONG) {
                    throw new AvroSerdeException("Unexpected Avro schema for Date TypeInfo: " + recordSchema.getType());
                }
                // If a time zone is found in file metadata (property name: writer.time.zone), convert the
                // timestamp to that (writer) time zone in order to emulate time zone agnostic behavior.
                // If not, then the file was written by an older version of hive, so we convert the timestamp
                // to the server's (reader) time zone for backwards compatibility reasons - unless the
                // session level configuration hive.avro.timestamp.skip.conversion is set to true, in which
                // case we assume it was written by a time zone agnostic writer, so we don't convert it.
                final boolean skipUTCConversion;
                if (configuration != null) {
                    skipUTCConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_SKIP_CONVERSION);
                } else {
                    skipUTCConversion = HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_SKIP_CONVERSION.defaultBoolVal;
                }
                final boolean legacyConversion;
                if (writerZoneConversionLegacy != null) {
                    legacyConversion = writerZoneConversionLegacy;
                } else if (writerTimezone != null) {
                    legacyConversion = false;
                } else if (configuration != null) {
                    legacyConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_LEGACY_CONVERSION_ENABLED);
                } else {
                    legacyConversion = HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_LEGACY_CONVERSION_ENABLED.defaultBoolVal;
                }
                ZoneId convertToTimeZone;
                if (writerTimezone != null) {
                    convertToTimeZone = writerTimezone;
                } else if (skipUTCConversion) {
                    convertToTimeZone = ZoneOffset.UTC;
                } else {
                    convertToTimeZone = TimeZone.getDefault().toZoneId();
                }
                final boolean skipProlepticConversion;
                if (writerProleptic != null) {
                    skipProlepticConversion = writerProleptic;
                } else {
                    if (configuration != null) {
                        skipProlepticConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT);
                    } else {
                        skipProlepticConversion = HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.defaultBoolVal;
                    }
                }
                Timestamp timestamp = TimestampTZUtil.convertTimestampToZone(Timestamp.ofEpochMilli((Long) datum), ZoneOffset.UTC, convertToTimeZone, legacyConversion);
                if (!skipProlepticConversion) {
                    timestamp = Timestamp.ofEpochMilli(CalendarUtils.convertTimeToProleptic(timestamp.toEpochMilli()));
                }
                return timestamp;
            }
        default:
            return datum;
    }
}

Example 57 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class TestGenericUDFTrunc method testDateWritableToDateWithQuarterFormat.

@Test
public void testDateWritableToDateWithQuarterFormat() throws HiveException {
    GenericUDFTrunc udf = new GenericUDFTrunc();
    ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableDateObjectInspector;
    ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] initArgs = { valueOI0, valueOI1 };
    DeferredObject valueObjFmt = new DeferredJavaObject(new Text("Q"));
    DeferredObject valueObj0;
    DeferredObject[] evalArgs;
    // test date string
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2014-01-01")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2014-01-14")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2014-01-31")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2014-02-02")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2014-02-28")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2016-02-03")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2016-02-28")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2016-02-29")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2016-05-11")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-04-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2016-07-01")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-07-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritableV2(Date.valueOf("2016-12-31")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-10-01", udf, initArgs, evalArgs);
}

Example 58 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class TestGenericUDFOPMinus method testDateMinusIntervalDayTime.

@Test
public void testDateMinusIntervalDayTime() throws Exception {
    GenericUDFOPMinus udf = new GenericUDFOPMinus();
    DateWritableV2 left = new DateWritableV2(Date.valueOf("2001-01-01"));
    HiveIntervalDayTimeWritable right = new HiveIntervalDayTimeWritable(HiveIntervalDayTime.valueOf("1 0:0:0.555"));
    ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableDateObjectInspector, PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector };
    DeferredObject[] args = { new DeferredJavaObject(left), new DeferredJavaObject(right) };
    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
    TimestampWritableV2 res = (TimestampWritableV2) udf.evaluate(args);
    Assert.assertEquals(Timestamp.valueOf("2000-12-30 23:59:59.445"), res.getTimestamp());
}

Example 59 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class TestGenericUDFOPMinus method testDateMinusIntervalYearMonth.

@Test
public void testDateMinusIntervalYearMonth() throws Exception {
    GenericUDFOPMinus udf = new GenericUDFOPMinus();
    DateWritableV2 left = new DateWritableV2(Date.valueOf("2004-02-15"));
    HiveIntervalYearMonthWritable right = new HiveIntervalYearMonthWritable(HiveIntervalYearMonth.valueOf("2-8"));
    ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableDateObjectInspector, PrimitiveObjectInspectorFactory.writableHiveIntervalYearMonthObjectInspector };
    DeferredObject[] args = { new DeferredJavaObject(left), new DeferredJavaObject(right) };
    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.dateTypeInfo, oi.getTypeInfo());
    DateWritableV2 res = (DateWritableV2) udf.evaluate(args);
    Assert.assertEquals(Date.valueOf("2001-06-15"), res.get());
}

Example 60 with DateWritableV2

use of org.apache.hadoop.hive.serde2.io.DateWritableV2 in project hive by apache.

the class TestGenericUDFOPPlus method testIntervalDayTimePlusDate.

@Test
public void testIntervalDayTimePlusDate() throws Exception {
    GenericUDFOPPlus udf = new GenericUDFOPPlus();
    HiveIntervalDayTimeWritable left = new HiveIntervalDayTimeWritable(HiveIntervalDayTime.valueOf("1 2:3:4.567"));
    DateWritableV2 right = new DateWritableV2(Date.valueOf("2001-01-01"));
    ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableHiveIntervalDayTimeObjectInspector, PrimitiveObjectInspectorFactory.writableDateObjectInspector };
    DeferredObject[] args = { new DeferredJavaObject(left), new DeferredJavaObject(right) };
    // Date + day-time interval = timestamp
    PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
    Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
    TimestampWritableV2 res = (TimestampWritableV2) udf.evaluate(args);
    Assert.assertEquals(Timestamp.valueOf("2001-01-02 2:3:4.567"), res.getTimestamp());
}