Search in sources :

Example 21 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class VerifyFastRow method serializeWrite.

public static void serializeWrite(SerializeWrite serializeWrite, TypeInfo typeInfo, Object object) throws IOException {
    if (object == null) {
        serializeWrite.writeNull();
        return;
    }
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                switch(primitiveTypeInfo.getPrimitiveCategory()) {
                    case BOOLEAN:
                        {
                            boolean value = ((BooleanWritable) object).get();
                            serializeWrite.writeBoolean(value);
                        }
                        break;
                    case BYTE:
                        {
                            byte value = ((ByteWritable) object).get();
                            serializeWrite.writeByte(value);
                        }
                        break;
                    case SHORT:
                        {
                            short value = ((ShortWritable) object).get();
                            serializeWrite.writeShort(value);
                        }
                        break;
                    case INT:
                        {
                            int value = ((IntWritable) object).get();
                            serializeWrite.writeInt(value);
                        }
                        break;
                    case LONG:
                        {
                            long value = ((LongWritable) object).get();
                            serializeWrite.writeLong(value);
                        }
                        break;
                    case FLOAT:
                        {
                            float value = ((FloatWritable) object).get();
                            serializeWrite.writeFloat(value);
                        }
                        break;
                    case DOUBLE:
                        {
                            double value = ((DoubleWritable) object).get();
                            serializeWrite.writeDouble(value);
                        }
                        break;
                    case STRING:
                        {
                            Text value = (Text) object;
                            byte[] stringBytes = value.getBytes();
                            int stringLength = stringBytes.length;
                            serializeWrite.writeString(stringBytes, 0, stringLength);
                        }
                        break;
                    case CHAR:
                        {
                            HiveChar value = ((HiveCharWritable) object).getHiveChar();
                            serializeWrite.writeHiveChar(value);
                        }
                        break;
                    case VARCHAR:
                        {
                            HiveVarchar value = ((HiveVarcharWritable) object).getHiveVarchar();
                            serializeWrite.writeHiveVarchar(value);
                        }
                        break;
                    case DECIMAL:
                        {
                            HiveDecimal value = ((HiveDecimalWritable) object).getHiveDecimal();
                            DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                            serializeWrite.writeHiveDecimal(value, decTypeInfo.scale());
                        }
                        break;
                    case DATE:
                        {
                            Date value = ((DateWritableV2) object).get();
                            serializeWrite.writeDate(value);
                        }
                        break;
                    case TIMESTAMP:
                        {
                            Timestamp value = ((TimestampWritableV2) object).getTimestamp();
                            serializeWrite.writeTimestamp(value);
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonth value = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
                            serializeWrite.writeHiveIntervalYearMonth(value);
                        }
                        break;
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTime value = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
                            serializeWrite.writeHiveIntervalDayTime(value);
                        }
                        break;
                    case BINARY:
                        {
                            BytesWritable byteWritable = (BytesWritable) object;
                            byte[] binaryBytes = byteWritable.getBytes();
                            int length = byteWritable.getLength();
                            serializeWrite.writeBinary(binaryBytes, 0, length);
                        }
                        break;
                    default:
                        throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory().name());
                }
            }
            break;
        case LIST:
            {
                ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
                TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo();
                ArrayList<Object> elements = (ArrayList<Object>) object;
                serializeWrite.beginList(elements);
                boolean isFirst = true;
                for (Object elementObject : elements) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateList();
                    }
                    if (elementObject == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, elementTypeInfo, elementObject);
                    }
                }
                serializeWrite.finishList();
            }
            break;
        case MAP:
            {
                MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
                TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo();
                TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo();
                Map<Object, Object> hashMap = (Map<Object, Object>) object;
                serializeWrite.beginMap(hashMap);
                boolean isFirst = true;
                for (Map.Entry<Object, Object> entry : hashMap.entrySet()) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateKeyValuePair();
                    }
                    if (entry.getKey() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, keyTypeInfo, entry.getKey());
                    }
                    serializeWrite.separateKey();
                    if (entry.getValue() == null) {
                        serializeWrite.writeNull();
                    } else {
                        serializeWrite(serializeWrite, valueTypeInfo, entry.getValue());
                    }
                }
                serializeWrite.finishMap();
            }
            break;
        case STRUCT:
            {
                StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
                List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
                List<Object> fieldValues = (List<Object>) object;
                final int size = fieldValues.size();
                serializeWrite.beginStruct(fieldValues);
                boolean isFirst = true;
                for (int i = 0; i < size; i++) {
                    if (isFirst) {
                        isFirst = false;
                    } else {
                        serializeWrite.separateStruct();
                    }
                    serializeWrite(serializeWrite, fieldTypeInfos.get(i), fieldValues.get(i));
                }
                serializeWrite.finishStruct();
            }
            break;
        case UNION:
            {
                UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
                List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
                final int size = fieldTypeInfos.size();
                StandardUnionObjectInspector.StandardUnion standardUnion = (StandardUnionObjectInspector.StandardUnion) object;
                byte tag = standardUnion.getTag();
                serializeWrite.beginUnion(tag);
                serializeWrite(serializeWrite, fieldTypeInfos.get(tag), standardUnion.getObject());
                serializeWrite.finishUnion();
            }
            break;
        default:
            throw new Error("Unknown category " + typeInfo.getCategory().name());
    }
}
Also used : StandardUnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardUnionObjectInspector) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) ArrayList(java.util.ArrayList) DoubleWritable(org.apache.hadoop.hive.serde2.io.DoubleWritable) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) Timestamp(org.apache.hadoop.hive.common.type.Timestamp) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) ArrayList(java.util.ArrayList) List(java.util.List) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.hive.serde2.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) HiveIntervalDayTime(org.apache.hadoop.hive.common.type.HiveIntervalDayTime) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) HiveVarcharWritable(org.apache.hadoop.hive.serde2.io.HiveVarcharWritable) DateWritableV2(org.apache.hadoop.hive.serde2.io.DateWritableV2) Text(org.apache.hadoop.io.Text) HiveIntervalDayTimeWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) HiveIntervalYearMonthWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable) TimestampWritableV2(org.apache.hadoop.hive.serde2.io.TimestampWritableV2) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) StructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo) VarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo) CharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo) Date(org.apache.hadoop.hive.common.type.Date) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) FloatWritable(org.apache.hadoop.io.FloatWritable) HiveIntervalYearMonth(org.apache.hadoop.hive.common.type.HiveIntervalYearMonth) BooleanWritable(org.apache.hadoop.io.BooleanWritable) ListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo) MapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) UnionTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo)

Example 22 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class VerifyFastRow method doVerifyDeserializeRead.

public static void doVerifyDeserializeRead(DeserializeRead deserializeRead, TypeInfo typeInfo, Object object, boolean isNull) throws IOException {
    if (isNull) {
        if (object != null) {
            TestCase.fail("Field reports null but object is not null (class " + object.getClass().getName() + ", " + object.toString() + ")");
        }
        return;
    } else if (object == null) {
        TestCase.fail("Field report not null but object is null");
    }
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                switch(primitiveTypeInfo.getPrimitiveCategory()) {
                    case BOOLEAN:
                        {
                            boolean value = deserializeRead.currentBoolean;
                            if (!(object instanceof BooleanWritable)) {
                                TestCase.fail("Boolean expected writable not Boolean");
                            }
                            boolean expected = ((BooleanWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case BYTE:
                        {
                            byte value = deserializeRead.currentByte;
                            if (!(object instanceof ByteWritable)) {
                                TestCase.fail("Byte expected writable not Byte");
                            }
                            byte expected = ((ByteWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                            }
                        }
                        break;
                    case SHORT:
                        {
                            short value = deserializeRead.currentShort;
                            if (!(object instanceof ShortWritable)) {
                                TestCase.fail("Short expected writable not Short");
                            }
                            short expected = ((ShortWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case INT:
                        {
                            int value = deserializeRead.currentInt;
                            if (!(object instanceof IntWritable)) {
                                TestCase.fail("Integer expected writable not Integer");
                            }
                            int expected = ((IntWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case LONG:
                        {
                            long value = deserializeRead.currentLong;
                            if (!(object instanceof LongWritable)) {
                                TestCase.fail("Long expected writable not Long");
                            }
                            Long expected = ((LongWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case FLOAT:
                        {
                            float value = deserializeRead.currentFloat;
                            if (!(object instanceof FloatWritable)) {
                                TestCase.fail("Float expected writable not Float");
                            }
                            float expected = ((FloatWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case DOUBLE:
                        {
                            double value = deserializeRead.currentDouble;
                            if (!(object instanceof DoubleWritable)) {
                                TestCase.fail("Double expected writable not Double");
                            }
                            double expected = ((DoubleWritable) object).get();
                            if (value != expected) {
                                TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
                            }
                        }
                        break;
                    case STRING:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            String expected = ((Text) object).toString();
                            if (!string.equals(expected)) {
                                TestCase.fail("String field mismatch (expected '" + expected + "' found '" + string + "')");
                            }
                        }
                        break;
                    case CHAR:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
                            HiveChar expected = ((HiveCharWritable) object).getHiveChar();
                            if (!hiveChar.equals(expected)) {
                                TestCase.fail("Char field mismatch (expected '" + expected + "' found '" + hiveChar + "')");
                            }
                        }
                        break;
                    case VARCHAR:
                        {
                            byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            Text text = new Text(stringBytes);
                            String string = text.toString();
                            HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
                            HiveVarchar expected = ((HiveVarcharWritable) object).getHiveVarchar();
                            if (!hiveVarchar.equals(expected)) {
                                TestCase.fail("Varchar field mismatch (expected '" + expected + "' found '" + hiveVarchar + "')");
                            }
                        }
                        break;
                    case DECIMAL:
                        {
                            HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
                            if (value == null) {
                                TestCase.fail("Decimal field evaluated to NULL");
                            }
                            HiveDecimal expected = ((HiveDecimalWritable) object).getHiveDecimal();
                            if (!value.equals(expected)) {
                                DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                                int precision = decimalTypeInfo.getPrecision();
                                int scale = decimalTypeInfo.getScale();
                                TestCase.fail("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                            }
                        }
                        break;
                    case DATE:
                        {
                            Date value = deserializeRead.currentDateWritable.get();
                            Date expected = ((DateWritableV2) object).get();
                            if (!value.equals(expected)) {
                                TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case TIMESTAMP:
                        {
                            Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
                            Timestamp expected = ((TimestampWritableV2) object).getTimestamp();
                            if (!value.equals(expected)) {
                                TestCase.fail("Timestamp field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case INTERVAL_YEAR_MONTH:
                        {
                            HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
                            HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
                            if (!value.equals(expected)) {
                                TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case INTERVAL_DAY_TIME:
                        {
                            HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
                            HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
                            if (!value.equals(expected)) {
                                TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                            }
                        }
                        break;
                    case BINARY:
                        {
                            byte[] byteArray = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                            BytesWritable bytesWritable = (BytesWritable) object;
                            byte[] expected = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                            if (byteArray.length != expected.length) {
                                TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                            }
                            for (int b = 0; b < byteArray.length; b++) {
                                if (byteArray[b] != expected[b]) {
                                    TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                                }
                            }
                        }
                        break;
                    default:
                        throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
                }
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
            throw new Error("Complex types need to be handled separately");
        default:
            throw new Error("Unknown category " + typeInfo.getCategory());
    }
}
Also used : HiveChar(org.apache.hadoop.hive.common.type.HiveChar) DoubleWritable(org.apache.hadoop.hive.serde2.io.DoubleWritable) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) Timestamp(org.apache.hadoop.hive.common.type.Timestamp) PrimitiveTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo) HiveDecimal(org.apache.hadoop.hive.common.type.HiveDecimal) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.hive.serde2.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) HiveIntervalDayTime(org.apache.hadoop.hive.common.type.HiveIntervalDayTime) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) HiveCharWritable(org.apache.hadoop.hive.serde2.io.HiveCharWritable) HiveVarcharWritable(org.apache.hadoop.hive.serde2.io.HiveVarcharWritable) DateWritableV2(org.apache.hadoop.hive.serde2.io.DateWritableV2) Text(org.apache.hadoop.io.Text) HiveIntervalDayTimeWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalDayTimeWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) HiveIntervalYearMonthWritable(org.apache.hadoop.hive.serde2.io.HiveIntervalYearMonthWritable) TimestampWritableV2(org.apache.hadoop.hive.serde2.io.TimestampWritableV2) Date(org.apache.hadoop.hive.common.type.Date) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) FloatWritable(org.apache.hadoop.io.FloatWritable) HiveIntervalYearMonth(org.apache.hadoop.hive.common.type.HiveIntervalYearMonth) BooleanWritable(org.apache.hadoop.io.BooleanWritable)

Example 23 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class VectorColumnGroupGenerator method generateRowColumnValue.

private void generateRowColumnValue(int rowIndex, int columnIndex, Random random) {
    GenerateType generateType = generateTypes[columnIndex];
    GenerateCategory category = generateType.getCategory();
    boolean allowNulls = generateType.getAllowNulls();
    if (allowNulls && random.nextInt(100) < 5) {
        isNullArrays[columnIndex][rowIndex] = true;
        return;
    }
    Object array = arrays[columnIndex];
    switch(category) {
        case BOOLEAN:
            {
                boolean value = random.nextBoolean();
                ((boolean[]) array)[rowIndex] = value;
            }
            break;
        case BYTE:
            {
                byte value = (byte) (random.nextBoolean() ? -random.nextInt(-((int) Byte.MIN_VALUE) + 1) : random.nextInt((int) Byte.MAX_VALUE + 1));
                ((byte[]) array)[rowIndex] = value;
            }
            break;
        case SHORT:
            {
                short value = (short) (random.nextBoolean() ? -random.nextInt(-((int) Short.MIN_VALUE) + 1) : random.nextInt((int) Short.MAX_VALUE + 1));
                ((short[]) array)[rowIndex] = value;
            }
            break;
        case INT:
            {
                int value = random.nextInt();
                ((int[]) array)[rowIndex] = value;
            }
            break;
        case LONG:
            {
                long value = random.nextLong();
                ((long[]) array)[rowIndex] = value;
            }
            break;
        case FLOAT:
            {
                float value = random.nextLong();
                ((float[]) array)[rowIndex] = value;
            }
            break;
        case DOUBLE:
            {
                double value = random.nextLong();
                ((double[]) array)[rowIndex] = value;
            }
            break;
        case STRING:
            {
                String value = RandomTypeUtil.getRandString(random);
                ((String[]) array)[rowIndex] = value;
            }
            break;
        case BINARY:
            {
                byte[] value = RandomTypeUtil.getRandBinary(random, 10);
                ((byte[][]) array)[rowIndex] = value;
            }
            break;
        case DATE:
            {
                Date value = RandomTypeUtil.getRandDate(random);
                ((Date[]) array)[rowIndex] = value;
            }
            break;
        case TIMESTAMP:
            {
                Timestamp value = RandomTypeUtil.getRandTimestamp(random).toSqlTimestamp();
                ((Timestamp[]) array)[rowIndex] = value;
            }
            break;
        case CHAR:
            {
                // UNDONE: Use CharTypeInfo.maxLength
                HiveChar value = new HiveChar(RandomTypeUtil.getRandString(random), 10);
                ((HiveChar[]) array)[rowIndex] = value;
            }
            break;
        case VARCHAR:
            {
                // UNDONE: Use VarcharTypeInfo.maxLength
                HiveVarchar value = new HiveVarchar(RandomTypeUtil.getRandString(random), 10);
                ((HiveVarchar[]) array)[rowIndex] = value;
            }
            break;
        case DECIMAL:
            {
                HiveDecimalWritable value = new HiveDecimalWritable(RandomTypeUtil.getRandHiveDecimal(random));
                ((HiveDecimalWritable[]) array)[rowIndex] = value;
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
        default:
    }
}
Also used : GenerateType(org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) GenerateCategory(org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType.GenerateCategory) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) Timestamp(java.sql.Timestamp) Date(org.apache.hadoop.hive.common.type.Date)

Example 24 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class VectorColumnGroupGenerator method fillDownRowColumnValue.

private void fillDownRowColumnValue(int rowIndex, int columnIndex, int seriesCount, Random random) {
    GenerateType generateType = generateTypes[columnIndex];
    GenerateCategory category = generateType.getCategory();
    boolean allowNulls = generateType.getAllowNulls();
    Object array = arrays[columnIndex];
    boolean[] isNull = isNullArrays[columnIndex];
    if (allowNulls && isNull[rowIndex]) {
        for (int i = 1; i < seriesCount; i++) {
            isNull[rowIndex + i] = true;
        }
        return;
    }
    switch(category) {
        case BOOLEAN:
            {
                boolean[] booleanArray = ((boolean[]) array);
                boolean value = booleanArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    booleanArray[rowIndex + i] = value;
                }
            }
            break;
        case BYTE:
            {
                byte[] byteArray = ((byte[]) array);
                byte value = byteArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    byteArray[rowIndex + i] = value;
                }
            }
            break;
        case SHORT:
            {
                short[] shortArray = ((short[]) array);
                short value = shortArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    shortArray[rowIndex + i] = value;
                }
            }
            break;
        case INT:
            {
                int[] intArray = ((int[]) array);
                int value = intArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    intArray[rowIndex + i] = value;
                }
            }
            break;
        case LONG:
            {
                long[] longArray = ((long[]) array);
                long value = longArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    longArray[rowIndex + i] = value;
                }
            }
            break;
        case FLOAT:
            {
                float[] floatArray = ((float[]) array);
                float value = floatArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    floatArray[rowIndex + i] = value;
                }
            }
            break;
        case DOUBLE:
            {
                double[] doubleArray = ((double[]) array);
                double value = doubleArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    doubleArray[rowIndex + i] = value;
                }
            }
            break;
        case STRING:
            {
                String[] stringArray = ((String[]) array);
                String value = stringArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    stringArray[rowIndex + i] = value;
                }
            }
            break;
        case BINARY:
            {
                byte[][] byteArrayArray = ((byte[][]) array);
                byte[] value = byteArrayArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    byteArrayArray[rowIndex + i] = value;
                }
            }
            break;
        case DATE:
            {
                Date[] dateArray = ((Date[]) array);
                Date value = dateArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    dateArray[rowIndex + i] = value;
                }
            }
            break;
        case TIMESTAMP:
            {
                Timestamp[] timestampArray = ((Timestamp[]) array);
                Timestamp value = timestampArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    timestampArray[rowIndex + i] = value;
                }
            }
            break;
        case CHAR:
            {
                HiveChar[] hiveCharArray = ((HiveChar[]) array);
                HiveChar value = hiveCharArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    hiveCharArray[rowIndex + i] = value;
                }
            }
            break;
        case VARCHAR:
            {
                HiveVarchar[] hiveVarcharArray = ((HiveVarchar[]) array);
                HiveVarchar value = hiveVarcharArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    hiveVarcharArray[rowIndex + i] = value;
                }
            }
            break;
        case DECIMAL:
            {
                HiveDecimalWritable[] hiveDecimalWritableArray = ((HiveDecimalWritable[]) array);
                HiveDecimalWritable value = hiveDecimalWritableArray[rowIndex];
                for (int i = 1; i < seriesCount; i++) {
                    hiveDecimalWritableArray[rowIndex + i] = value;
                }
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
        default:
    }
}
Also used : GenerateType(org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) GenerateCategory(org.apache.hadoop.hive.ql.exec.vector.util.batchgen.VectorBatchGenerator.GenerateType.GenerateCategory) HiveChar(org.apache.hadoop.hive.common.type.HiveChar) HiveVarchar(org.apache.hadoop.hive.common.type.HiveVarchar) Timestamp(java.sql.Timestamp) Date(org.apache.hadoop.hive.common.type.Date)

Example 25 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class TestParquetTimestampUtils method testJulianDay.

@Test
public void testJulianDay() {
    // check if May 23, 1968 is Julian Day 2440000
    GregorianCalendar cal = new GregorianCalendar();
    cal.setTimeZone(TimeZone.getTimeZone("GMT"));
    cal.setGregorianChange(new java.util.Date(Long.MIN_VALUE));
    cal.set(Calendar.YEAR, 1968);
    cal.set(Calendar.MONTH, Calendar.MAY);
    cal.set(Calendar.DAY_OF_MONTH, 23);
    cal.set(Calendar.HOUR_OF_DAY, 0);
    Timestamp ts = Timestamp.ofEpochMilli(cal.getTimeInMillis());
    NanoTime nt = NanoTimeUtils.getNanoTime(ts, TimeZone.getDefault().toZoneId(), false);
    Assert.assertEquals(nt.getJulianDay(), 2440000);
    Timestamp tsFetched = NanoTimeUtils.getTimestamp(nt, TimeZone.getDefault().toZoneId());
    Assert.assertEquals(tsFetched, ts);
    // check if 30 Julian Days between Jan 1, 2005 and Jan 31, 2005.
    GregorianCalendar cal1 = new GregorianCalendar();
    cal1.setTimeZone(TimeZone.getTimeZone("GMT"));
    cal1.setGregorianChange(new java.util.Date(Long.MIN_VALUE));
    cal1.set(Calendar.YEAR, 2005);
    cal1.set(Calendar.MONTH, Calendar.JANUARY);
    cal1.set(Calendar.DAY_OF_MONTH, 1);
    cal1.set(Calendar.HOUR_OF_DAY, 0);
    Timestamp ts1 = Timestamp.ofEpochMilli(cal1.getTimeInMillis());
    NanoTime nt1 = NanoTimeUtils.getNanoTime(ts1, TimeZone.getDefault().toZoneId(), false);
    Timestamp ts1Fetched = NanoTimeUtils.getTimestamp(nt1, TimeZone.getDefault().toZoneId());
    Assert.assertEquals(ts1Fetched, ts1);
    GregorianCalendar cal2 = new GregorianCalendar();
    cal2.setTimeZone(TimeZone.getTimeZone("UTC"));
    cal2.setGregorianChange(new java.util.Date(Long.MIN_VALUE));
    cal2.set(Calendar.YEAR, 2005);
    cal2.set(Calendar.MONTH, Calendar.JANUARY);
    cal2.set(Calendar.DAY_OF_MONTH, 31);
    cal2.set(Calendar.HOUR_OF_DAY, 0);
    Timestamp ts2 = Timestamp.ofEpochMilli(cal2.getTimeInMillis());
    NanoTime nt2 = NanoTimeUtils.getNanoTime(ts2, TimeZone.getDefault().toZoneId(), false);
    Timestamp ts2Fetched = NanoTimeUtils.getTimestamp(nt2, TimeZone.getDefault().toZoneId());
    Assert.assertEquals(ts2Fetched, ts2);
    Assert.assertEquals(nt2.getJulianDay() - nt1.getJulianDay(), 30);
    // check if 730517 Julian Days between Jan 1, 0005 and Jan 31, 2005.
    // This method used to test Julian Days between Jan 1, 2005 BCE and Jan 1, 2005 CE. Since BCE
    // timestamps are not supported, both dates were changed to CE.
    cal1 = new GregorianCalendar();
    cal1.setTimeZone(TimeZone.getTimeZone("GMT"));
    cal1.setGregorianChange(new java.util.Date(Long.MIN_VALUE));
    cal1.set(Calendar.YEAR, 0005);
    cal1.set(Calendar.MONTH, Calendar.JANUARY);
    cal1.set(Calendar.DAY_OF_MONTH, 1);
    cal1.set(Calendar.HOUR_OF_DAY, 0);
    ts1 = Timestamp.ofEpochMilli(cal1.getTimeInMillis());
    nt1 = NanoTimeUtils.getNanoTime(ts1, TimeZone.getDefault().toZoneId(), false);
    ts1Fetched = NanoTimeUtils.getTimestamp(nt1, TimeZone.getDefault().toZoneId());
    Assert.assertEquals(ts1Fetched, ts1);
    cal2 = new GregorianCalendar();
    cal2.setTimeZone(TimeZone.getTimeZone("UTC"));
    cal2.setGregorianChange(new java.util.Date(Long.MIN_VALUE));
    cal2.set(Calendar.YEAR, 2005);
    cal2.set(Calendar.MONTH, Calendar.JANUARY);
    cal2.set(Calendar.DAY_OF_MONTH, 31);
    cal2.set(Calendar.HOUR_OF_DAY, 0);
    ts2 = Timestamp.ofEpochMilli(cal2.getTimeInMillis());
    nt2 = NanoTimeUtils.getNanoTime(ts2, TimeZone.getDefault().toZoneId(), false);
    ts2Fetched = NanoTimeUtils.getTimestamp(nt2, TimeZone.getDefault().toZoneId());
    Assert.assertEquals(ts2Fetched, ts2);
    Assert.assertEquals(730517, nt2.getJulianDay() - nt1.getJulianDay());
    Date d1 = Date.ofEpochMilli(cal1.getTimeInMillis());
    Assert.assertEquals("0005-01-01", d1.toString());
    Date d2 = Date.ofEpochMilli(cal2.getTimeInMillis());
    Assert.assertEquals("2005-01-31", d2.toString());
}
Also used : NanoTime(org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime) GregorianCalendar(java.util.GregorianCalendar) Timestamp(org.apache.hadoop.hive.common.type.Timestamp) Date(org.apache.hadoop.hive.common.type.Date) Test(org.junit.Test)

Aggregations

Date (org.apache.hadoop.hive.common.type.Date)71 Timestamp (org.apache.hadoop.hive.common.type.Timestamp)26 DateWritableV2 (org.apache.hadoop.hive.serde2.io.DateWritableV2)21 HiveChar (org.apache.hadoop.hive.common.type.HiveChar)18 HiveVarchar (org.apache.hadoop.hive.common.type.HiveVarchar)18 HiveDecimal (org.apache.hadoop.hive.common.type.HiveDecimal)17 HiveDecimalWritable (org.apache.hadoop.hive.serde2.io.HiveDecimalWritable)15 PrimitiveTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo)14 Text (org.apache.hadoop.io.Text)14 Test (org.junit.Test)14 HiveIntervalYearMonth (org.apache.hadoop.hive.common.type.HiveIntervalYearMonth)13 BytesWritable (org.apache.hadoop.io.BytesWritable)12 HiveIntervalDayTime (org.apache.hadoop.hive.common.type.HiveIntervalDayTime)11 TimestampWritableV2 (org.apache.hadoop.hive.serde2.io.TimestampWritableV2)11 DecimalTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo)11 List (java.util.List)10 LongWritable (org.apache.hadoop.io.LongWritable)10 ByteWritable (org.apache.hadoop.hive.serde2.io.ByteWritable)9 DoubleWritable (org.apache.hadoop.hive.serde2.io.DoubleWritable)9 HiveCharWritable (org.apache.hadoop.hive.serde2.io.HiveCharWritable)9