Examples with HiveChar org.apache.hadoop.hive.common.type.HiveChar used on opensource projects

Example 66 with HiveChar

use of org.apache.hadoop.hive.common.type.HiveChar in project hive by apache.

the class AvroDeserializer method deserializePrimitive.

private Object deserializePrimitive(Object datum, Schema fileSchema, Schema recordSchema, PrimitiveTypeInfo columnType) throws AvroSerdeException {
    switch(columnType.getPrimitiveCategory()) {
        case STRING:
            // To workaround AvroUTF8
            return datum.toString();
        // and convert it to a string. Yay!
        case BINARY:
            if (recordSchema.getType() == Type.FIXED) {
                Fixed fixed = (Fixed) datum;
                return fixed.bytes();
            } else if (recordSchema.getType() == Type.BYTES) {
                return AvroSerdeUtils.getBytesFromByteBuffer((ByteBuffer) datum);
            } else {
                throw new AvroSerdeException("Unexpected Avro schema for Binary TypeInfo: " + recordSchema.getType());
            }
        case DECIMAL:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for decimal field. Reader schema is " + columnType);
            }
            int scale = 0;
            try {
                scale = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_SCALE);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex);
            }
            HiveDecimal dec = AvroSerdeUtils.getHiveDecimalFromByteBuffer((ByteBuffer) datum, scale);
            JavaHiveDecimalObjectInspector oi = (JavaHiveDecimalObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector((DecimalTypeInfo) columnType);
            return oi.set(null, dec);
        case CHAR:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for char field. Reader schema is " + columnType);
            }
            int maxLength = 0;
            try {
                maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain maxLength value for char field from file schema: " + fileSchema, ex);
            }
            String str = datum.toString();
            HiveChar hc = new HiveChar(str, maxLength);
            return hc;
        case VARCHAR:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for varchar field. Reader schema is " + columnType);
            }
            maxLength = 0;
            try {
                maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain maxLength value for varchar field from file schema: " + fileSchema, ex);
            }
            str = datum.toString();
            HiveVarchar hvc = new HiveVarchar(str, maxLength);
            return hvc;
        case DATE:
            {
                if (recordSchema.getType() != Type.INT) {
                    throw new AvroSerdeException("Unexpected Avro schema for Date TypeInfo: " + recordSchema.getType());
                }
                final boolean skipProlepticConversion;
                if (writerProleptic != null) {
                    skipProlepticConversion = writerProleptic;
                } else {
                    if (configuration != null) {
                        skipProlepticConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT);
                    } else {
                        skipProlepticConversion = HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.defaultBoolVal;
                    }
                }
                return Date.ofEpochMilli(DateWritableV2.daysToMillis(skipProlepticConversion ? (Integer) datum : CalendarUtils.convertDateToProleptic((Integer) datum)));
            }
        case TIMESTAMP:
            {
                if (recordSchema.getType() != Type.LONG) {
                    throw new AvroSerdeException("Unexpected Avro schema for Date TypeInfo: " + recordSchema.getType());
                }
                // If a time zone is found in file metadata (property name: writer.time.zone), convert the
                // timestamp to that (writer) time zone in order to emulate time zone agnostic behavior.
                // If not, then the file was written by an older version of hive, so we convert the timestamp
                // to the server's (reader) time zone for backwards compatibility reasons - unless the
                // session level configuration hive.avro.timestamp.skip.conversion is set to true, in which
                // case we assume it was written by a time zone agnostic writer, so we don't convert it.
                final boolean skipUTCConversion;
                if (configuration != null) {
                    skipUTCConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_SKIP_CONVERSION);
                } else {
                    skipUTCConversion = HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_SKIP_CONVERSION.defaultBoolVal;
                }
                final boolean legacyConversion;
                if (writerZoneConversionLegacy != null) {
                    legacyConversion = writerZoneConversionLegacy;
                } else if (writerTimezone != null) {
                    legacyConversion = false;
                } else if (configuration != null) {
                    legacyConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_LEGACY_CONVERSION_ENABLED);
                } else {
                    legacyConversion = HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_LEGACY_CONVERSION_ENABLED.defaultBoolVal;
                }
                ZoneId convertToTimeZone;
                if (writerTimezone != null) {
                    convertToTimeZone = writerTimezone;
                } else if (skipUTCConversion) {
                    convertToTimeZone = ZoneOffset.UTC;
                } else {
                    convertToTimeZone = TimeZone.getDefault().toZoneId();
                }
                final boolean skipProlepticConversion;
                if (writerProleptic != null) {
                    skipProlepticConversion = writerProleptic;
                } else {
                    if (configuration != null) {
                        skipProlepticConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT);
                    } else {
                        skipProlepticConversion = HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.defaultBoolVal;
                    }
                }
                Timestamp timestamp = TimestampTZUtil.convertTimestampToZone(Timestamp.ofEpochMilli((Long) datum), ZoneOffset.UTC, convertToTimeZone, legacyConversion);
                if (!skipProlepticConversion) {
                    timestamp = Timestamp.ofEpochMilli(CalendarUtils.convertTimeToProleptic(timestamp.toEpochMilli()));
                }
                return timestamp;
            }
        default:
            return datum;
    }
}

Example 67 with HiveChar

use of org.apache.hadoop.hive.common.type.HiveChar in project hive by apache.

the class TestSearchArgumentImpl method testBuilderComplexTypes2.

@Test
public void testBuilderComplexTypes2() throws Exception {
    SearchArgumentImpl sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startAnd().lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("2005-3-12")).lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()).equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")).end().build();
    assertEquals("leaf-0 = (LESS_THAN x 2005-03-12), " + "leaf-1 = (LESS_THAN_EQUALS y hi        ), " + "leaf-2 = (EQUALS z 1), " + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toOldString());
    sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startNot().startOr().isNull("x", PredicateLeaf.Type.LONG).between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")).in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L).nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()).end().end().build();
    assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " + "leaf-2 = (IN z 1 2 3), " + "leaf-3 = (NULL_SAFE_EQUALS a stinger), " + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toOldString());
}

Example 68 with HiveChar

use of org.apache.hadoop.hive.common.type.HiveChar in project hive by apache.

the class TestSearchArgumentImpl method testBuilderComplexTypes.

@Test
public void testBuilderComplexTypes() throws Exception {
    SearchArgumentImpl sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startAnd().lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("1970-1-11")).lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()).equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")).end().build();
    assertEquals("leaf-0 = (LESS_THAN x 1970-01-11), " + "leaf-1 = (LESS_THAN_EQUALS y hi        ), " + "leaf-2 = (EQUALS z 1), " + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toOldString());
    sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startNot().startOr().isNull("x", PredicateLeaf.Type.LONG).between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")).in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L).nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()).end().end().build();
    assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " + "leaf-2 = (IN z 1 2 3), " + "leaf-3 = (NULL_SAFE_EQUALS a stinger), " + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toOldString());
}

Example 69 with HiveChar

use of org.apache.hadoop.hive.common.type.HiveChar in project hive by apache.

the class TestParquetFilterPredicate method testFilterCharColumnGreaterThan.

@Test
public void testFilterCharColumnGreaterThan() throws Exception {
    SearchArgument sarg = SearchArgumentFactory.newBuilder().startNot().lessThanEquals("a", PredicateLeaf.Type.STRING, new HiveChar("apple", 10).toString()).end().build();
    MessageType schema = MessageTypeParser.parseMessageType("message test {required binary a;}");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("a", TypeInfoFactory.getCharTypeInfo(10));
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes);
    String expected = "not(lteq(a, Binary{\"apple\"}))";
    assertEquals(expected, p.toString());
}

Example 70 with HiveChar

use of org.apache.hadoop.hive.common.type.HiveChar in project hive by apache.

the class TestParquetFilterPredicate method testFilterCharColumnBetween.

@Test
public void testFilterCharColumnBetween() throws Exception {
    SearchArgument sarg = SearchArgumentFactory.newBuilder().between("a", PredicateLeaf.Type.STRING, new HiveChar("apple", 10).toString(), new HiveChar("pear", 10).toString()).build();
    MessageType schema = MessageTypeParser.parseMessageType("message test {required binary a;}");
    Map<String, TypeInfo> columnTypes = new HashMap<>();
    columnTypes.put("a", TypeInfoFactory.getCharTypeInfo(10));
    FilterPredicate p = ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema, columnTypes);
    String expected = "and(lteq(a, Binary{\"pear\"}), not(lt(a, Binary{\"apple\"})))";
    assertEquals(expected, p.toString());
}