Examples with HiveVarchar org.apache.hadoop.hive.common.type.HiveVarchar used on opensource projects

Example 61 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class VectorVerifyFast method serializeWrite.

public static void serializeWrite(SerializeWrite serializeWrite, TypeInfo typeInfo, Object object) throws IOException {
    if (object == null) {
        serializeWrite.writeNull();
        return;
    }
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                switch(primitiveTypeInfo.getPrimitiveCategory()) {
                    case BOOLEAN:
                        {
                            boolean value = ((BooleanWritable) object).get();
                            serializeWrite.writeBoolean(value);
                        }
                        break;
                    case BYTE:
                        {
                            byte value = ((ByteWritable) object).get();
                            serializeWrite.writeByte(value);
                        }
                        break;
                    case SHORT:
                        {
                            short value = ((ShortWritable) object).get();
                            serializeWrite.writeShort(value);
                        }
                        break;
                    case INT:
                        {
                            int value = ((IntWritable) object).get();
                            serializeWrite.writeInt(value);
                        }
                        break;
                    case LONG:
                        {
                            long value = ((LongWritable) object).get();
                            serializeWrite.writeLong(value);
                        }
                        break;
                    case FLOAT:
                        {
                            float value = ((FloatWritable) object).get();
                            serializeWrite.writeFloat(value);
                        }
                        break;
                    case DOUBLE:
                        {
                            double value = ((DoubleWritable) object).get();
                            serializeWrite.writeDouble(value);
                        }
                        break;
                    case STRING:
                        {
                            Text value = (Text) object;
                            byte[] bytes = value.getBytes();
                            int byteLength = value.getLength();
                            serializeWrite.writeString(bytes, 0, byteLength);
                        }
                        break;
                    case CHAR:
                        {
                            HiveChar value = ((HiveCharWritable) object).getHiveChar();
                            serializeWrite.writeHiveChar(value);
                        }
                        break;
                    case VARCHAR:
                        {
                            HiveVarchar value = ((HiveVarcharWritable) object).getHiveVarchar();
                            serializeWrite.writeHiveVarchar(value);
                        }
                        break;
                    case DECIMAL:
                        {
                            HiveDecimal value = ((HiveDecimalWritable) object).getHiveDecimal();
                            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                            serializeWrite.writeHiveDecimal(value, decTypeInfo.scale());
                        }
                        break;
                    case DATE:
                        {
                            Date value = ((DateWritableV2) object).get();
                            serializeWrite.writeDate(value);
                        }
                        break;
                    case TIMESTAMP:
                        {
                            Timestamp value = ((TimestampWritableV2) object).getTimestamp();
                            serializeWrite.writeTimestamp(value);
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonth value = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
                            serializeWrite.writeHiveIntervalYearMonth(value);
                        }
                        break;
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTime value = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
                            serializeWrite.writeHiveIntervalDayTime(value);
                        }
                        break;
                    case BINARY:
                        {
                            BytesWritable byteWritable = (BytesWritable) object;
                            byte[] binaryBytes = byteWritable.getBytes();
                            int length = byteWritable.getLength();
                            serializeWrite.writeBinary(binaryBytes, 0, length);
                        }
                        break;
                    default:
                        throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory().name());
                }
            }
            break;
        case LIST:
            {
                ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
                TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo();
                ArrayList<Object> elements = (ArrayList<Object>) object;
                serializeWrite.beginList(elements);
                boolean isFirst = true;
                for (Object elementObject : elements) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateList();
                    }
                    if (elementObject == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, elementTypeInfo, elementObject);
                    }
                }
                serializeWrite.finishList();
            }
            break;
        case MAP:
            {
                MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
                TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo();
                TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo();
                HashMap<Object, Object> hashMap = (HashMap<Object, Object>) object;
                serializeWrite.beginMap(hashMap);
                boolean isFirst = true;
                for (Map.Entry<Object, Object> entry : hashMap.entrySet()) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateKeyValuePair();
                    }
                    if (entry.getKey() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, keyTypeInfo, entry.getKey());
                    }
                    serializeWrite.separateKey();
                    if (entry.getValue() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, valueTypeInfo, entry.getValue());
                    }
                }
                serializeWrite.finishMap();
            }
            break;
        case STRUCT:
            {
                StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
                List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
                List<Object> fieldValues = (List<Object>) object;
                final int size = fieldValues.size();
                serializeWrite.beginStruct(fieldValues);
                boolean isFirst = true;
                for (int i = 0; i < size; i++) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateStruct();
                    }
                    serializeWrite(serializeWrite, fieldTypeInfos.get(i), fieldValues.get(i));
                }
                serializeWrite.finishStruct();
            }
            break;
        case UNION:
            {
                UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
                List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
                final int size = fieldTypeInfos.size();
                StandardUnionObjectInspector.StandardUnion standardUnion = (StandardUnionObjectInspector.StandardUnion) object;
                byte tag = standardUnion.getTag();
                serializeWrite.beginUnion(tag);
                serializeWrite(serializeWrite, fieldTypeInfos.get(tag), standardUnion.getObject());
                serializeWrite.finishUnion();
            }
            break;
        default:
            throw new Error("Unknown category " + typeInfo.getCategory().name());
    }
}

Example 62 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class AvroSerializer method serializePrimitive.

private Object serializePrimitive(TypeInfo typeInfo, PrimitiveObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException {
    switch(fieldOI.getPrimitiveCategory()) {
        case BINARY:
            if (schema.getType() == Type.BYTES) {
                return AvroSerdeUtils.getBufferFromBytes((byte[]) fieldOI.getPrimitiveJavaObject(structFieldData));
            } else if (schema.getType() == Type.FIXED) {
                Fixed fixed = new GenericData.Fixed(schema, (byte[]) fieldOI.getPrimitiveJavaObject(structFieldData));
                return fixed;
            } else {
                throw new AvroSerdeException("Unexpected Avro schema for Binary TypeInfo: " + schema.getType());
            }
        case DECIMAL:
            HiveDecimal dec = (HiveDecimal) fieldOI.getPrimitiveJavaObject(structFieldData);
            return AvroSerdeUtils.getBufferFromDecimal(dec, ((DecimalTypeInfo) typeInfo).scale());
        case CHAR:
            HiveChar ch = (HiveChar) fieldOI.getPrimitiveJavaObject(structFieldData);
            return ch.getStrippedValue();
        case VARCHAR:
            HiveVarchar vc = (HiveVarchar) fieldOI.getPrimitiveJavaObject(structFieldData);
            return vc.getValue();
        case DATE:
            Date date = ((DateObjectInspector) fieldOI).getPrimitiveJavaObject(structFieldData);
            return defaultProleptic ? date.toEpochDay() : CalendarUtils.convertDateToHybrid(date.toEpochDay());
        case TIMESTAMP:
            Timestamp timestamp = ((TimestampObjectInspector) fieldOI).getPrimitiveJavaObject(structFieldData);
            long millis = defaultProleptic ? timestamp.toEpochMilli() : CalendarUtils.convertTimeToHybrid(timestamp.toEpochMilli());
            timestamp = TimestampTZUtil.convertTimestampToZone(Timestamp.ofEpochMilli(millis), TimeZone.getDefault().toZoneId(), ZoneOffset.UTC, legacyConversion);
            return timestamp.toEpochMilli();
        case UNKNOWN:
            throw new AvroSerdeException("Received UNKNOWN primitive category.");
        case VOID:
            return null;
        default:
            // All other primitive types are simple
            return fieldOI.getPrimitiveJavaObject(structFieldData);
    }
}

Example 63 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class AvroDeserializer method deserializePrimitive.

private Object deserializePrimitive(Object datum, Schema fileSchema, Schema recordSchema, PrimitiveTypeInfo columnType) throws AvroSerdeException {
    switch(columnType.getPrimitiveCategory()) {
        case STRING:
            // To workaround AvroUTF8
            return datum.toString();
        // and convert it to a string. Yay!
        case BINARY:
            if (recordSchema.getType() == Type.FIXED) {
                Fixed fixed = (Fixed) datum;
                return fixed.bytes();
            } else if (recordSchema.getType() == Type.BYTES) {
                return AvroSerdeUtils.getBytesFromByteBuffer((ByteBuffer) datum);
            } else {
                throw new AvroSerdeException("Unexpected Avro schema for Binary TypeInfo: " + recordSchema.getType());
            }
        case DECIMAL:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for decimal field. Reader schema is " + columnType);
            }
            int scale = 0;
            try {
                scale = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_SCALE);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex);
            }
            HiveDecimal dec = AvroSerdeUtils.getHiveDecimalFromByteBuffer((ByteBuffer) datum, scale);
            JavaHiveDecimalObjectInspector oi = (JavaHiveDecimalObjectInspector) PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector((DecimalTypeInfo) columnType);
            return oi.set(null, dec);
        case CHAR:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for char field. Reader schema is " + columnType);
            }
            int maxLength = 0;
            try {
                maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain maxLength value for char field from file schema: " + fileSchema, ex);
            }
            String str = datum.toString();
            HiveChar hc = new HiveChar(str, maxLength);
            return hc;
        case VARCHAR:
            if (fileSchema == null) {
                throw new AvroSerdeException("File schema is missing for varchar field. Reader schema is " + columnType);
            }
            maxLength = 0;
            try {
                maxLength = AvroSerdeUtils.getIntFromSchema(fileSchema, AvroSerDe.AVRO_PROP_MAX_LENGTH);
            } catch (Exception ex) {
                throw new AvroSerdeException("Failed to obtain maxLength value for varchar field from file schema: " + fileSchema, ex);
            }
            str = datum.toString();
            HiveVarchar hvc = new HiveVarchar(str, maxLength);
            return hvc;
        case DATE:
            {
                if (recordSchema.getType() != Type.INT) {
                    throw new AvroSerdeException("Unexpected Avro schema for Date TypeInfo: " + recordSchema.getType());
                }
                final boolean skipProlepticConversion;
                if (writerProleptic != null) {
                    skipProlepticConversion = writerProleptic;
                } else {
                    if (configuration != null) {
                        skipProlepticConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT);
                    } else {
                        skipProlepticConversion = HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.defaultBoolVal;
                    }
                }
                return Date.ofEpochMilli(DateWritableV2.daysToMillis(skipProlepticConversion ? (Integer) datum : CalendarUtils.convertDateToProleptic((Integer) datum)));
            }
        case TIMESTAMP:
            {
                if (recordSchema.getType() != Type.LONG) {
                    throw new AvroSerdeException("Unexpected Avro schema for Date TypeInfo: " + recordSchema.getType());
                }
                // If a time zone is found in file metadata (property name: writer.time.zone), convert the
                // timestamp to that (writer) time zone in order to emulate time zone agnostic behavior.
                // If not, then the file was written by an older version of hive, so we convert the timestamp
                // to the server's (reader) time zone for backwards compatibility reasons - unless the
                // session level configuration hive.avro.timestamp.skip.conversion is set to true, in which
                // case we assume it was written by a time zone agnostic writer, so we don't convert it.
                final boolean skipUTCConversion;
                if (configuration != null) {
                    skipUTCConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_SKIP_CONVERSION);
                } else {
                    skipUTCConversion = HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_SKIP_CONVERSION.defaultBoolVal;
                }
                final boolean legacyConversion;
                if (writerZoneConversionLegacy != null) {
                    legacyConversion = writerZoneConversionLegacy;
                } else if (writerTimezone != null) {
                    legacyConversion = false;
                } else if (configuration != null) {
                    legacyConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_LEGACY_CONVERSION_ENABLED);
                } else {
                    legacyConversion = HiveConf.ConfVars.HIVE_AVRO_TIMESTAMP_LEGACY_CONVERSION_ENABLED.defaultBoolVal;
                }
                ZoneId convertToTimeZone;
                if (writerTimezone != null) {
                    convertToTimeZone = writerTimezone;
                } else if (skipUTCConversion) {
                    convertToTimeZone = ZoneOffset.UTC;
                } else {
                    convertToTimeZone = TimeZone.getDefault().toZoneId();
                }
                final boolean skipProlepticConversion;
                if (writerProleptic != null) {
                    skipProlepticConversion = writerProleptic;
                } else {
                    if (configuration != null) {
                        skipProlepticConversion = HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT);
                    } else {
                        skipProlepticConversion = HiveConf.ConfVars.HIVE_AVRO_PROLEPTIC_GREGORIAN_DEFAULT.defaultBoolVal;
                    }
                }
                Timestamp timestamp = TimestampTZUtil.convertTimestampToZone(Timestamp.ofEpochMilli((Long) datum), ZoneOffset.UTC, convertToTimeZone, legacyConversion);
                if (!skipProlepticConversion) {
                    timestamp = Timestamp.ofEpochMilli(CalendarUtils.convertTimeToProleptic(timestamp.toEpochMilli()));
                }
                return timestamp;
            }
        default:
            return datum;
    }
}

Example 64 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestSearchArgumentImpl method testBuilderComplexTypes2.

@Test
public void testBuilderComplexTypes2() throws Exception {
    SearchArgumentImpl sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startAnd().lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("2005-3-12")).lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()).equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")).end().build();
    assertEquals("leaf-0 = (LESS_THAN x 2005-03-12), " + "leaf-1 = (LESS_THAN_EQUALS y hi        ), " + "leaf-2 = (EQUALS z 1), " + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toOldString());
    sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startNot().startOr().isNull("x", PredicateLeaf.Type.LONG).between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")).in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L).nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()).end().end().build();
    assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " + "leaf-2 = (IN z 1 2 3), " + "leaf-3 = (NULL_SAFE_EQUALS a stinger), " + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toOldString());
}

Example 65 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestSearchArgumentImpl method testBuilderComplexTypes.

@Test
public void testBuilderComplexTypes() throws Exception {
    SearchArgumentImpl sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startAnd().lessThan("x", PredicateLeaf.Type.DATE, Date.valueOf("1970-1-11")).lessThanEquals("y", PredicateLeaf.Type.STRING, new HiveChar("hi", 10).toString()).equals("z", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("1.0")).end().build();
    assertEquals("leaf-0 = (LESS_THAN x 1970-01-11), " + "leaf-1 = (LESS_THAN_EQUALS y hi        ), " + "leaf-2 = (EQUALS z 1), " + "expr = (and leaf-0 leaf-1 leaf-2)", sarg.toOldString());
    sarg = (SearchArgumentImpl) SearchArgumentFactory.newBuilder().startNot().startOr().isNull("x", PredicateLeaf.Type.LONG).between("y", PredicateLeaf.Type.DECIMAL, new HiveDecimalWritable("10"), new HiveDecimalWritable("20.0")).in("z", PredicateLeaf.Type.LONG, 1L, 2L, 3L).nullSafeEquals("a", PredicateLeaf.Type.STRING, new HiveVarchar("stinger", 100).toString()).end().end().build();
    assertEquals("leaf-0 = (IS_NULL x), " + "leaf-1 = (BETWEEN y 10 20), " + "leaf-2 = (IN z 1 2 3), " + "leaf-3 = (NULL_SAFE_EQUALS a stinger), " + "expr = (and (not leaf-0) (not leaf-1) (not leaf-2) (not leaf-3))", sarg.toOldString());
}