Examples with DateWritable org.apache.hadoop.hive.serde2.io.DateWritable used on opensource projects

Example 16 with DateWritable

use of org.apache.hadoop.hive.serde2.io.DateWritable in project hive by apache.

the class TestGenericUDFDate method testStringToDate.

public void testStringToDate() throws HiveException {
    GenericUDFDate udf = new GenericUDFDate();
    ObjectInspector valueOI = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] arguments = { valueOI };
    udf.initialize(arguments);
    DeferredObject valueObj = new DeferredJavaObject(new Text("2009-07-30"));
    DeferredObject[] args = { valueObj };
    DateWritable output = (DateWritable) udf.evaluate(args);
    assertEquals("to_date() test for STRING failed ", "2009-07-30", output.toString());
    // Try with null args
    DeferredObject[] nullArgs = { new DeferredJavaObject(null) };
    output = (DateWritable) udf.evaluate(nullArgs);
    assertNull("to_date() with null STRING", output);
}

Example 17 with DateWritable

use of org.apache.hadoop.hive.serde2.io.DateWritable in project hive by apache.

the class TestGenericUDFTrunc method testDateWritableToDateWithMonthFormat.

public void testDateWritableToDateWithMonthFormat() throws HiveException {
    GenericUDFTrunc udf = new GenericUDFTrunc();
    ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableDateObjectInspector;
    ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] initArgs = { valueOI0, valueOI1 };
    DeferredObject valueObjFmt = new DeferredJavaObject(new Text("MM"));
    DeferredObject valueObj0;
    DeferredObject[] evalArgs;
    // test date string
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-01-01")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-01-14")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-01-31")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-02-02")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-02-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-02-28")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-02-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2016-02-03")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-02-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2016-02-28")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-02-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2016-02-29")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-02-01", udf, initArgs, evalArgs);
}

Example 18 with DateWritable

use of org.apache.hadoop.hive.serde2.io.DateWritable in project hive by apache.

the class TestGenericUDFTrunc method testDateWritableToDateWithYearFormat.

public void testDateWritableToDateWithYearFormat() throws HiveException {
    GenericUDFTrunc udf = new GenericUDFTrunc();
    ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableDateObjectInspector;
    ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector;
    ObjectInspector[] initArgs = { valueOI0, valueOI1 };
    DeferredObject valueObjFmt = new DeferredJavaObject(new Text("YY"));
    DeferredObject valueObj0;
    DeferredObject[] evalArgs;
    // test date string
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-01-01")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-01-14")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-01-31")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-02-02")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2014-02-28")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2014-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2016-02-03")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2016-02-28")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-01-01", udf, initArgs, evalArgs);
    valueObj0 = new DeferredJavaObject(new DateWritable(Date.valueOf("2016-02-29")));
    evalArgs = new DeferredObject[] { valueObj0, valueObjFmt };
    runAndVerify("2016-01-01", udf, initArgs, evalArgs);
}

Example 19 with DateWritable

use of org.apache.hadoop.hive.serde2.io.DateWritable in project hive by apache.

the class VerifyFast method verifyDeserializeRead.

public static void verifyDeserializeRead(DeserializeRead deserializeRead, PrimitiveTypeInfo primitiveTypeInfo, Writable writable) throws IOException {
    boolean isNull;
    isNull = !deserializeRead.readNextField();
    if (isNull) {
        if (writable != null) {
            TestCase.fail("Field reports null but object is not null (class " + writable.getClass().getName() + ", " + writable.toString() + ")");
        }
        return;
    } else if (writable == null) {
        TestCase.fail("Field report not null but object is null");
    }
    switch(primitiveTypeInfo.getPrimitiveCategory()) {
        case BOOLEAN:
            {
                boolean value = deserializeRead.currentBoolean;
                if (!(writable instanceof BooleanWritable)) {
                    TestCase.fail("Boolean expected writable not Boolean");
                }
                boolean expected = ((BooleanWritable) writable).get();
                if (value != expected) {
                    TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
                }
            }
            break;
        case BYTE:
            {
                byte value = deserializeRead.currentByte;
                if (!(writable instanceof ByteWritable)) {
                    TestCase.fail("Byte expected writable not Byte");
                }
                byte expected = ((ByteWritable) writable).get();
                if (value != expected) {
                    TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                }
            }
            break;
        case SHORT:
            {
                short value = deserializeRead.currentShort;
                if (!(writable instanceof ShortWritable)) {
                    TestCase.fail("Short expected writable not Short");
                }
                short expected = ((ShortWritable) writable).get();
                if (value != expected) {
                    TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
                }
            }
            break;
        case INT:
            {
                int value = deserializeRead.currentInt;
                if (!(writable instanceof IntWritable)) {
                    TestCase.fail("Integer expected writable not Integer");
                }
                int expected = ((IntWritable) writable).get();
                if (value != expected) {
                    TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
                }
            }
            break;
        case LONG:
            {
                long value = deserializeRead.currentLong;
                if (!(writable instanceof LongWritable)) {
                    TestCase.fail("Long expected writable not Long");
                }
                Long expected = ((LongWritable) writable).get();
                if (value != expected) {
                    TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
                }
            }
            break;
        case FLOAT:
            {
                float value = deserializeRead.currentFloat;
                if (!(writable instanceof FloatWritable)) {
                    TestCase.fail("Float expected writable not Float");
                }
                float expected = ((FloatWritable) writable).get();
                if (value != expected) {
                    TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
                }
            }
            break;
        case DOUBLE:
            {
                double value = deserializeRead.currentDouble;
                if (!(writable instanceof DoubleWritable)) {
                    TestCase.fail("Double expected writable not Double");
                }
                double expected = ((DoubleWritable) writable).get();
                if (value != expected) {
                    TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
                }
            }
            break;
        case STRING:
            {
                byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                Text text = new Text(stringBytes);
                String string = text.toString();
                String expected = ((Text) writable).toString();
                if (!string.equals(expected)) {
                    TestCase.fail("String field mismatch (expected '" + expected + "' found '" + string + "')");
                }
            }
            break;
        case CHAR:
            {
                byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                Text text = new Text(stringBytes);
                String string = text.toString();
                HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
                HiveChar expected = ((HiveCharWritable) writable).getHiveChar();
                if (!hiveChar.equals(expected)) {
                    TestCase.fail("Char field mismatch (expected '" + expected + "' found '" + hiveChar + "')");
                }
            }
            break;
        case VARCHAR:
            {
                byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                Text text = new Text(stringBytes);
                String string = text.toString();
                HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
                HiveVarchar expected = ((HiveVarcharWritable) writable).getHiveVarchar();
                if (!hiveVarchar.equals(expected)) {
                    TestCase.fail("Varchar field mismatch (expected '" + expected + "' found '" + hiveVarchar + "')");
                }
            }
            break;
        case DECIMAL:
            {
                HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
                if (value == null) {
                    TestCase.fail("Decimal field evaluated to NULL");
                }
                HiveDecimal expected = ((HiveDecimalWritable) writable).getHiveDecimal();
                if (!value.equals(expected)) {
                    DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                    int precision = decimalTypeInfo.getPrecision();
                    int scale = decimalTypeInfo.getScale();
                    TestCase.fail("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                }
            }
            break;
        case DATE:
            {
                Date value = deserializeRead.currentDateWritable.get();
                Date expected = ((DateWritable) writable).get();
                if (!value.equals(expected)) {
                    TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                }
            }
            break;
        case TIMESTAMP:
            {
                Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
                Timestamp expected = ((TimestampWritable) writable).getTimestamp();
                if (!value.equals(expected)) {
                    TestCase.fail("Timestamp field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                }
            }
            break;
        case INTERVAL_YEAR_MONTH:
            {
                HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
                HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) writable).getHiveIntervalYearMonth();
                if (!value.equals(expected)) {
                    TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                }
            }
            break;
        case INTERVAL_DAY_TIME:
            {
                HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
                HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) writable).getHiveIntervalDayTime();
                if (!value.equals(expected)) {
                    TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                }
            }
            break;
        case BINARY:
            {
                byte[] byteArray = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                BytesWritable bytesWritable = (BytesWritable) writable;
                byte[] expected = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                if (byteArray.length != expected.length) {
                    TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                }
                for (int b = 0; b < byteArray.length; b++) {
                    if (byteArray[b] != expected[b]) {
                        TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
                    }
                }
            }
            break;
        default:
            throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
    }
}

Example 20 with DateWritable

use of org.apache.hadoop.hive.serde2.io.DateWritable in project hive by apache.

the class TestVectorizedORCReader method checkVectorizedReader.

private void checkVectorizedReader() throws Exception {
    Reader vreader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf));
    Reader reader = OrcFile.createReader(testFilePath, OrcFile.readerOptions(conf));
    RecordReaderImpl vrr = (RecordReaderImpl) vreader.rows();
    RecordReaderImpl rr = (RecordReaderImpl) reader.rows();
    VectorizedRowBatch batch = reader.getSchema().createRowBatch();
    OrcStruct row = null;
    // Check Vectorized ORC reader against ORC row reader
    while (vrr.nextBatch(batch)) {
        for (int i = 0; i < batch.size; i++) {
            row = (OrcStruct) rr.next(row);
            for (int j = 0; j < batch.cols.length; j++) {
                Object a = (row.getFieldValue(j));
                ColumnVector cv = batch.cols[j];
                // if the value is repeating, use row 0
                int rowId = cv.isRepeating ? 0 : i;
                // make sure the null flag agrees
                if (a == null) {
                    Assert.assertEquals(true, !cv.noNulls && cv.isNull[rowId]);
                } else if (a instanceof BooleanWritable) {
                    // Boolean values are stores a 1's and 0's, so convert and compare
                    Long temp = (long) (((BooleanWritable) a).get() ? 1 : 0);
                    long b = ((LongColumnVector) cv).vector[rowId];
                    Assert.assertEquals(temp.toString(), Long.toString(b));
                } else if (a instanceof TimestampWritable) {
                    // Timestamps are stored as long, so convert and compare
                    TimestampWritable t = ((TimestampWritable) a);
                    TimestampColumnVector tcv = ((TimestampColumnVector) cv);
                    Assert.assertEquals(t.getTimestamp(), tcv.asScratchTimestamp(rowId));
                } else if (a instanceof DateWritable) {
                    // Dates are stored as long, so convert and compare
                    DateWritable adt = (DateWritable) a;
                    long b = ((LongColumnVector) cv).vector[rowId];
                    Assert.assertEquals(adt.get().getTime(), DateWritable.daysToMillis((int) b));
                } else if (a instanceof HiveDecimalWritable) {
                    // Decimals are stored as BigInteger, so convert and compare
                    HiveDecimalWritable dec = (HiveDecimalWritable) a;
                    HiveDecimalWritable b = ((DecimalColumnVector) cv).vector[i];
                    Assert.assertEquals(dec, b);
                } else if (a instanceof DoubleWritable) {
                    double b = ((DoubleColumnVector) cv).vector[rowId];
                    assertEquals(a.toString(), Double.toString(b));
                } else if (a instanceof Text) {
                    BytesColumnVector bcv = (BytesColumnVector) cv;
                    Text b = new Text();
                    b.set(bcv.vector[rowId], bcv.start[rowId], bcv.length[rowId]);
                    assertEquals(a, b);
                } else if (a instanceof IntWritable || a instanceof LongWritable || a instanceof ByteWritable || a instanceof ShortWritable) {
                    assertEquals(a.toString(), Long.toString(((LongColumnVector) cv).vector[rowId]));
                } else {
                    assertEquals("huh", a.getClass().getName());
                }
            }
        }
        // Check repeating
        Assert.assertEquals(false, batch.cols[0].isRepeating);
        Assert.assertEquals(false, batch.cols[1].isRepeating);
        Assert.assertEquals(false, batch.cols[2].isRepeating);
        Assert.assertEquals(true, batch.cols[3].isRepeating);
        Assert.assertEquals(false, batch.cols[4].isRepeating);
        Assert.assertEquals(false, batch.cols[5].isRepeating);
        Assert.assertEquals(false, batch.cols[6].isRepeating);
        Assert.assertEquals(false, batch.cols[7].isRepeating);
        Assert.assertEquals(false, batch.cols[8].isRepeating);
        Assert.assertEquals(false, batch.cols[9].isRepeating);
        // Check non null
        Assert.assertEquals(false, batch.cols[0].noNulls);
        Assert.assertEquals(false, batch.cols[1].noNulls);
        Assert.assertEquals(true, batch.cols[2].noNulls);
        Assert.assertEquals(true, batch.cols[3].noNulls);
        Assert.assertEquals(false, batch.cols[4].noNulls);
        Assert.assertEquals(false, batch.cols[5].noNulls);
        Assert.assertEquals(false, batch.cols[6].noNulls);
        Assert.assertEquals(false, batch.cols[7].noNulls);
        Assert.assertEquals(false, batch.cols[8].noNulls);
        Assert.assertEquals(false, batch.cols[9].noNulls);
    }
    Assert.assertEquals(false, rr.nextBatch(batch));
}