Examples with Date org.apache.hadoop.hive.common.type.Date used on opensource projects

Example 61 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class VectorColumnGroupGenerator method populateBatchColumn.

private void populateBatchColumn(VectorizedRowBatch batch, int logicalColumnIndex, int size) {
    int columnNum = columnNums[logicalColumnIndex];
    ColumnVector colVector = batch.cols[columnNum];
    GenerateType generateType = generateTypes[logicalColumnIndex];
    GenerateCategory category = generateType.getCategory();
    boolean allowNulls = generateType.getAllowNulls();
    boolean[] isNull = isNullArrays[logicalColumnIndex];
    if (allowNulls) {
        for (int i = 0; i < size; i++) {
            if (isNull[i]) {
                colVector.isNull[i] = true;
                colVector.noNulls = false;
            }
        }
    }
    Object array = arrays[logicalColumnIndex];
    switch(category) {
        case BOOLEAN:
            {
                boolean[] booleanArray = ((boolean[]) array);
                long[] vector = ((LongColumnVector) colVector).vector;
                for (int i = 0; i < size; i++) {
                    if (isNull[i]) {
                        vector[i] = 0;
                    } else {
                        vector[i] = (booleanArray[i] ? 1 : 0);
                    }
                }
            }
            break;
        case BYTE:
            {
                byte[] byteArray = ((byte[]) array);
                long[] vector = ((LongColumnVector) colVector).vector;
                for (int i = 0; i < size; i++) {
                    if (isNull[i]) {
                        vector[i] = 0;
                    } else {
                        vector[i] = byteArray[i];
                    }
                }
            }
            break;
        case SHORT:
            {
                short[] shortArray = ((short[]) array);
                long[] vector = ((LongColumnVector) colVector).vector;
                for (int i = 0; i < size; i++) {
                    if (isNull[i]) {
                        vector[i] = 0;
                    } else {
                        vector[i] = shortArray[i];
                    }
                }
            }
            break;
        case INT:
            {
                int[] intArray = ((int[]) array);
                long[] vector = ((LongColumnVector) colVector).vector;
                for (int i = 0; i < size; i++) {
                    if (isNull[i]) {
                        vector[i] = 0;
                    } else {
                        vector[i] = intArray[i];
                    }
                }
            }
            break;
        case LONG:
            {
                long[] longArray = ((long[]) array);
                long[] vector = ((LongColumnVector) colVector).vector;
                for (int i = 0; i < size; i++) {
                    if (isNull[i]) {
                        vector[i] = 0;
                    } else {
                        vector[i] = longArray[i];
                    }
                }
            }
            break;
        case FLOAT:
            {
                float[] floatArray = ((float[]) array);
                double[] vector = ((DoubleColumnVector) colVector).vector;
                for (int i = 0; i < size; i++) {
                    if (isNull[i]) {
                        vector[i] = 0;
                    } else {
                        vector[i] = floatArray[i];
                    }
                }
            }
            break;
        case DOUBLE:
            {
                double[] doubleArray = ((double[]) array);
                double[] vector = ((DoubleColumnVector) colVector).vector;
                for (int i = 0; i < size; i++) {
                    if (isNull[i]) {
                        vector[i] = 0;
                    } else {
                        vector[i] = doubleArray[i];
                    }
                }
            }
            break;
        case STRING:
            {
                String[] stringArray = ((String[]) array);
                BytesColumnVector bytesColVec = ((BytesColumnVector) colVector);
                for (int i = 0; i < size; i++) {
                    if (!isNull[i]) {
                        byte[] bytes = stringArray[i].getBytes();
                        bytesColVec.setVal(i, bytes);
                    }
                }
            }
            break;
        case BINARY:
            {
                byte[][] byteArrayArray = ((byte[][]) array);
                BytesColumnVector bytesColVec = ((BytesColumnVector) colVector);
                for (int i = 0; i < size; i++) {
                    if (!isNull[i]) {
                        byte[] bytes = byteArrayArray[i];
                        bytesColVec.setVal(i, bytes);
                    }
                }
            }
            break;
        case DATE:
            {
                Date[] dateArray = ((Date[]) array);
                LongColumnVector longColVec = ((LongColumnVector) colVector);
                for (int i = 0; i < size; i++) {
                    if (!isNull[i]) {
                        Date date = dateArray[i];
                        longColVec.vector[i] = DateWritableV2.dateToDays(date);
                    }
                }
            }
            break;
        case TIMESTAMP:
            {
                Timestamp[] timestampArray = ((Timestamp[]) array);
                TimestampColumnVector timestampColVec = ((TimestampColumnVector) colVector);
                for (int i = 0; i < size; i++) {
                    if (!isNull[i]) {
                        Timestamp timestamp = timestampArray[i];
                        timestampColVec.set(i, timestamp);
                    }
                }
            }
            break;
        case CHAR:
            {
                HiveChar[] hiveCharArray = ((HiveChar[]) array);
                BytesColumnVector bytesColVec = ((BytesColumnVector) colVector);
                for (int i = 0; i < size; i++) {
                    if (!isNull[i]) {
                        byte[] bytes = hiveCharArray[i].getValue().getBytes();
                        bytesColVec.setVal(i, bytes);
                    }
                }
            }
            break;
        case VARCHAR:
            {
                HiveVarchar[] hiveCharArray = ((HiveVarchar[]) array);
                BytesColumnVector bytesColVec = ((BytesColumnVector) colVector);
                for (int i = 0; i < size; i++) {
                    if (!isNull[i]) {
                        byte[] bytes = hiveCharArray[i].getValue().getBytes();
                        bytesColVec.setVal(i, bytes);
                    }
                }
            }
            break;
        case DECIMAL:
            {
                HiveDecimalWritable[] hiveDecimalWritableArray = ((HiveDecimalWritable[]) array);
                DecimalColumnVector decimalColVec = ((DecimalColumnVector) colVector);
                for (int i = 0; i < size; i++) {
                    if (!isNull[i]) {
                        HiveDecimalWritable decWritable = hiveDecimalWritableArray[i];
                        decimalColVec.set(i, decWritable);
                    }
                }
            }
            break;
        case LIST:
        case MAP:
        case STRUCT:
        case UNION:
        default:
            throw new RuntimeException("Unepected generate category " + category);
    }
}

Example 62 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class TestSemanticAnalyzer method checkNormalization.

public void checkNormalization(String colType, String originalColSpec, String result, Object colValue) throws SemanticException {
    final String colName = "col";
    Map<String, String> partSpec = new HashMap<String, String>();
    partSpec.put(colName, originalColSpec);
    BaseSemanticAnalyzer.normalizeColSpec(partSpec, colName, colType, originalColSpec, colValue);
    assertEquals(result, partSpec.get(colName));
    if (colValue instanceof Date) {
        DateWritableV2 dw = new DateWritableV2((Date) colValue);
        BaseSemanticAnalyzer.normalizeColSpec(partSpec, colName, colType, originalColSpec, dw);
        assertEquals(result, partSpec.get(colName));
    }
}

Example 63 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class VerifyLazy method lazyCompare.

public static boolean lazyCompare(TypeInfo typeInfo, Object lazyObject, Object expectedObject) {
    if (expectedObject == null) {
        if (lazyObject != null) {
            throw new RuntimeException("Expected object is null but object is not null " + lazyObject.toString() + " typeInfo " + typeInfo.toString());
        }
        return true;
    } else if (lazyObject == null) {
        throw new RuntimeException("Expected object is not null \"" + expectedObject.toString() + "\" typeInfo " + typeInfo.toString() + " but object is null");
    }
    if (lazyObject instanceof Writable) {
        if (!lazyObject.equals(expectedObject)) {
            throw new RuntimeException("Expected object " + expectedObject.toString() + " and actual object " + lazyObject.toString() + " is not equal typeInfo " + typeInfo.toString());
        }
        return true;
    }
    if (lazyObject instanceof LazyPrimitive) {
        Object primitiveObject = ((LazyPrimitive) lazyObject).getObject();
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
        switch(primitiveTypeInfo.getPrimitiveCategory()) {
            case BOOLEAN:
                {
                    if (!(primitiveObject instanceof LazyBoolean)) {
                        throw new RuntimeException("Expected LazyBoolean");
                    }
                    boolean value = ((LazyBoolean) primitiveObject).getWritableObject().get();
                    boolean expected = ((BooleanWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Boolean field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BYTE:
                {
                    if (!(primitiveObject instanceof LazyByte)) {
                        throw new RuntimeException("Expected LazyByte");
                    }
                    byte value = ((LazyByte) primitiveObject).getWritableObject().get();
                    byte expected = ((ByteWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                    }
                }
                break;
            case SHORT:
                {
                    if (!(primitiveObject instanceof LazyShort)) {
                        throw new RuntimeException("Expected LazyShort");
                    }
                    short value = ((LazyShort) primitiveObject).getWritableObject().get();
                    short expected = ((ShortWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Short field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INT:
                {
                    if (!(primitiveObject instanceof LazyInteger)) {
                        throw new RuntimeException("Expected LazyInteger");
                    }
                    int value = ((LazyInteger) primitiveObject).getWritableObject().get();
                    int expected = ((IntWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Int field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case LONG:
                {
                    if (!(primitiveObject instanceof LazyLong)) {
                        throw new RuntimeException("Expected LazyLong");
                    }
                    long value = ((LazyLong) primitiveObject).getWritableObject().get();
                    long expected = ((LongWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Long field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case FLOAT:
                {
                    if (!(primitiveObject instanceof LazyFloat)) {
                        throw new RuntimeException("Expected LazyFloat");
                    }
                    float value = ((LazyFloat) primitiveObject).getWritableObject().get();
                    float expected = ((FloatWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Float field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DOUBLE:
                {
                    if (!(primitiveObject instanceof LazyDouble)) {
                        throw new RuntimeException("Expected LazyDouble");
                    }
                    double value = ((LazyDouble) primitiveObject).getWritableObject().get();
                    double expected = ((DoubleWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Double field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case STRING:
                {
                    if (!(primitiveObject instanceof LazyString)) {
                        throw new RuntimeException("Text expected writable not Text");
                    }
                    Text value = ((LazyString) primitiveObject).getWritableObject();
                    Text expected = ((Text) expectedObject);
                    if (!value.equals(expected)) {
                        throw new RuntimeException("String field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case CHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveChar)) {
                        throw new RuntimeException("Expected LazyHiveChar");
                    }
                    HiveChar value = ((LazyHiveChar) primitiveObject).getWritableObject().getHiveChar();
                    HiveChar expected = ((HiveCharWritable) expectedObject).getHiveChar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveChar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case VARCHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveVarchar)) {
                        throw new RuntimeException("Expected LazyHiveVarchar");
                    }
                    HiveVarchar value = ((LazyHiveVarchar) primitiveObject).getWritableObject().getHiveVarchar();
                    HiveVarchar expected = ((HiveVarcharWritable) expectedObject).getHiveVarchar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveVarchar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case DECIMAL:
                {
                    if (!(primitiveObject instanceof LazyHiveDecimal)) {
                        throw new RuntimeException("Expected LazyDecimal");
                    }
                    HiveDecimal value = ((LazyHiveDecimal) primitiveObject).getWritableObject().getHiveDecimal();
                    HiveDecimal expected = ((HiveDecimalWritable) expectedObject).getHiveDecimal();
                    if (!value.equals(expected)) {
                        DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                        int precision = decimalTypeInfo.getPrecision();
                        int scale = decimalTypeInfo.getScale();
                        throw new RuntimeException("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                    }
                }
                break;
            case DATE:
                {
                    if (!(primitiveObject instanceof LazyDate)) {
                        throw new RuntimeException("Expected LazyDate");
                    }
                    Date value = ((LazyDate) primitiveObject).getWritableObject().get();
                    Date expected = ((DateWritableV2) expectedObject).get();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Date field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case TIMESTAMP:
                {
                    if (!(primitiveObject instanceof LazyTimestamp)) {
                        throw new RuntimeException("TimestampWritableV2 expected writable not TimestampWritableV2");
                    }
                    Timestamp value = ((LazyTimestamp) primitiveObject).getWritableObject().getTimestamp();
                    Timestamp expected = ((TimestampWritableV2) expectedObject).getTimestamp();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Timestamp field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalYearMonth)) {
                        throw new RuntimeException("Expected LazyHiveIntervalYearMonth");
                    }
                    HiveIntervalYearMonth value = ((LazyHiveIntervalYearMonth) primitiveObject).getWritableObject().getHiveIntervalYearMonth();
                    HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) expectedObject).getHiveIntervalYearMonth();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalYearMonth field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalDayTime)) {
                        throw new RuntimeException("Expected writable LazyHiveIntervalDayTime");
                    }
                    HiveIntervalDayTime value = ((LazyHiveIntervalDayTime) primitiveObject).getWritableObject().getHiveIntervalDayTime();
                    HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) expectedObject).getHiveIntervalDayTime();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalDayTime field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BINARY:
                {
                    if (!(primitiveObject instanceof LazyBinary)) {
                        throw new RuntimeException("Expected LazyBinary");
                    }
                    BytesWritable bytesWritable = ((LazyBinary) primitiveObject).getWritableObject();
                    byte[] value = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                    BytesWritable bytesWritableExpected = (BytesWritable) expectedObject;
                    byte[] expected = Arrays.copyOfRange(bytesWritableExpected.getBytes(), 0, bytesWritableExpected.getLength());
                    if (value.length != expected.length) {
                        throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                    }
                    for (int b = 0; b < value.length; b++) {
                        if (value[b] != expected[b]) {
                            throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                        }
                    }
                }
                break;
            default:
                throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
        }
    } else if (lazyObject instanceof LazyArray) {
        LazyArray lazyArray = (LazyArray) lazyObject;
        List<Object> list = lazyArray.getList();
        List<Object> expectedList = (List<Object>) expectedObject;
        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")" + " elementTypeInfo " + listTypeInfo.getListElementTypeInfo().toString());
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (typeInfo instanceof ListTypeInfo) {
        List<Object> list;
        if (lazyObject instanceof LazyBinaryArray) {
            list = ((LazyBinaryArray) lazyObject).getList();
        } else {
            list = (List<Object>) lazyObject;
        }
        List<Object> expectedList = (List<Object>) expectedObject;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")");
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (lazyObject instanceof LazyMap) {
        LazyMap lazyMap = (LazyMap) lazyObject;
        Map<Object, Object> map = lazyMap.getMap();
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (typeInfo instanceof MapTypeInfo) {
        Map<Object, Object> map;
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        if (lazyObject instanceof LazyBinaryMap) {
            map = ((LazyBinaryMap) lazyObject).getMap();
        } else {
            map = (Map<Object, Object>) lazyObject;
        }
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (lazyObject instanceof LazyStruct) {
        LazyStruct lazyStruct = (LazyStruct) lazyObject;
        List<Object> fields = lazyStruct.getFieldsAsList();
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (typeInfo instanceof StructTypeInfo) {
        ArrayList<Object> fields;
        if (lazyObject instanceof LazyBinaryStruct) {
            fields = ((LazyBinaryStruct) lazyObject).getFieldsAsList();
        } else {
            fields = (ArrayList<Object>) lazyObject;
        }
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (lazyObject instanceof LazyUnion) {
        LazyUnion union = (LazyUnion) lazyObject;
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        return lazyCompareUnion(unionTypeInfo, union, expectedUnion);
    } else if (typeInfo instanceof UnionTypeInfo) {
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        if (lazyObject instanceof LazyBinaryUnion) {
            return lazyCompareUnion(unionTypeInfo, (LazyBinaryUnion) lazyObject, expectedUnion);
        } else {
            return lazyCompareUnion(unionTypeInfo, (UnionObject) lazyObject, expectedUnion);
        }
    } else {
        System.err.println("Not implemented " + typeInfo.getClass().getName());
    }
    return true;
}

Example 64 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class PigHCatUtil method extractPigObject.

/**
 * Converts object from Hive's value system to Pig's value system
 * see HCatBaseStorer#getJavaObj() for Pig->Hive conversion
 * @param o object from Hive value system
 * @return object in Pig value system
 */
public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws Exception {
    /*Note that HCatRecordSerDe.serializePrimitiveField() will be called before this, thus some
    * type promotion/conversion may occur: e.g. Short to Integer.  We should refactor this so
    * that it's hapenning in one place per module/product that we are integrating with.
    * All Pig conversion should be done here, etc.*/
    if (o == null) {
        return null;
    }
    Object result;
    Type itemType = hfs.getType();
    switch(itemType) {
        case BINARY:
            result = new DataByteArray((byte[]) o);
            break;
        case STRUCT:
            result = transformToTuple((List<?>) o, hfs);
            break;
        case ARRAY:
            result = transformToBag((List<?>) o, hfs);
            break;
        case MAP:
            result = transformToPigMap((Map<?, ?>) o, hfs);
            break;
        case DECIMAL:
            result = ((HiveDecimal) o).bigDecimalValue();
            break;
        case CHAR:
            result = ((HiveChar) o).getValue();
            break;
        case VARCHAR:
            result = ((HiveVarchar) o).getValue();
            break;
        case DATE:
            /*java.sql.Date is weird.  It automatically adjusts it's millis value to be in the local TZ
      * e.g. d = new java.sql.Date(System.currentMillis()).toString() so if you do this just after
      * midnight in Palo Alto, you'll get yesterday's date printed out.*/
            Date d = (Date) o;
            result = new DateTime(d.getYear(), d.getMonth(), d.getDay(), 0, 0, DateTimeZone.UTC);
            break;
        case TIMESTAMP:
            /*DATA TRUNCATION!!!
       Timestamp may have nanos; we'll strip those away and create a Joda DateTime
       object in local TZ; This is arbitrary, since Hive value doesn't have any TZ notion, but
       we need to set something for TZ.
       Timestamp is consistently in GMT (unless you call toString() on it) so we use millis*/
            result = new DateTime(((Timestamp) o).toEpochMilli(), DateTimeZone.UTC);
            break;
        default:
            result = o;
            break;
    }
    return result;
}

Example 65 with Date

use of org.apache.hadoop.hive.common.type.Date in project hive by apache.

the class DateParser method parseDate.

/**
 * Obtains an instance of Date from a text string such as 2021-02-21.
 *
 * @param text the text to parse
 * @return The Date objects generated from parsing the text or null if the
 *         text could not be parsed
 * @throws NullPointerException if {@code text} is null
 */
public static Date parseDate(final String text) {
    Objects.requireNonNull(text);
    // Date is a mutable class; do not return cached value
    Date result = new Date();
    return (parseDate(text, result)) ? result : null;
}