use of org.apache.hadoop.hive.common.type.Date in project hive by apache.
the class TeradataBinaryDataInputStream method readDate.
/**
* Read DATE.
* The representation of date in Teradata binary format is:
* The Date D is a int with 4 bytes using little endian,
* The representation is (D+19000000).ToString -> YYYYMMDD,
* eg: Date 07 b2 01 00 -> 111111 in little endian -> 19111111 - > 1911.11.11.
* the null date will use 0 to pad.
*
* @return the date
* @throws IOException the io exception
* @throws ParseException the parse exception
*/
public Date readDate() throws IOException, ParseException {
int di = readInt();
if (di == 0) {
return null;
}
String dateString = String.valueOf(di + 19000000);
if (dateString.length() < DATE_STRING_LENGTH) {
dateString = StringUtils.leftPad(dateString, DATE_STRING_LENGTH, '0');
}
Date date = new Date();
date.setYear(Integer.parseInt(dateString.substring(0, 4)));
date.setMonth(Integer.parseInt(dateString.substring(4, 6)));
date.setDayOfMonth(Integer.parseInt(dateString.substring(6, 8)));
return date;
}
use of org.apache.hadoop.hive.common.type.Date in project hive by apache.
the class PrimitiveObjectInspectorUtils method getDate.
public static Date getDate(Object o, PrimitiveObjectInspector oi) {
if (o == null) {
return null;
}
Date result = null;
switch(oi.getPrimitiveCategory()) {
case VOID:
break;
case STRING:
StringObjectInspector soi = (StringObjectInspector) oi;
String s = soi.getPrimitiveJavaObject(o).trim();
if (s.length() == DATE_LENGTH) {
result = DateParser.parseDate(s);
} else {
Timestamp ts = getTimestampFromString(s);
if (ts != null) {
result = Date.ofEpochMilli(ts.toEpochMilli());
}
}
break;
case CHAR:
case VARCHAR:
{
String val = getString(o, oi).trim();
if (val.length() == DATE_LENGTH) {
result = DateParser.parseDate(val);
} else {
Timestamp ts = getTimestampFromString(val);
if (ts != null) {
result = Date.ofEpochMilli(ts.toEpochMilli());
}
}
break;
}
case DATE:
result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get();
break;
case TIMESTAMP:
result = DateWritableV2.timeToDate(((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getSeconds());
break;
case TIMESTAMPLOCALTZ:
String tstz = oi.getPrimitiveWritableObject(o).toString();
int divSpace = tstz.indexOf(' ');
if (divSpace == -1) {
return null;
}
result = Date.valueOf(tstz.substring(0, divSpace));
break;
default:
throw new RuntimeException("Cannot convert to Date from: " + oi.getTypeName());
}
return result;
}
use of org.apache.hadoop.hive.common.type.Date in project hive by apache.
the class TestGenericUDFLeast method testLeastDate.
@Test
public void testLeastDate() throws HiveException {
GenericUDFLeast udf = new GenericUDFLeast();
ObjectInspector[] arguments = new ObjectInspector[3];
for (int i = 0; i < arguments.length; i++) {
arguments[i] = PrimitiveObjectInspectorFactory.writableDateObjectInspector;
}
udf.initialize(arguments);
Date d1 = Date.valueOf("2015-03-20");
Date d2 = Date.valueOf("2015-03-21");
Date d3 = Date.valueOf("2014-03-20");
runAndVerify(new Date[] { d1, d2, d3 }, d3, udf);
runAndVerify(new Date[] { null, d2, d3 }, null, udf);
runAndVerify(new Date[] { d1, null, d3 }, null, udf);
runAndVerify(new Date[] { d1, d2, null }, null, udf);
runAndVerify(new Date[] { null, null, null }, null, udf);
}
use of org.apache.hadoop.hive.common.type.Date in project hive by apache.
the class RegexSerDe method deserialize.
@Override
public Object deserialize(Writable blob) throws SerDeException {
Text rowText = (Text) blob;
Matcher m = inputPattern.matcher(rowText.toString());
if (m.groupCount() != numColumns) {
throw new SerDeException("Number of matching groups doesn't match the number of columns");
}
// If do not match, ignore the line, return a row with all nulls.
if (!m.matches()) {
unmatchedRowsCount++;
if (!alreadyLoggedNoMatch) {
// Report the row if its the first time
log.warn("" + unmatchedRowsCount + " unmatched rows are found: " + rowText);
alreadyLoggedNoMatch = true;
}
return null;
}
// Otherwise, return the row.
for (int c = 0; c < numColumns; c++) {
try {
String t = m.group(c + 1);
TypeInfo typeInfo = getColumnTypes().get(c);
// Convert the column to the correct type when needed and set in row obj
PrimitiveTypeInfo pti = (PrimitiveTypeInfo) typeInfo;
switch(pti.getPrimitiveCategory()) {
case STRING:
row.set(c, t);
break;
case BYTE:
Byte b;
b = Byte.valueOf(t);
row.set(c, b);
break;
case SHORT:
Short s;
s = Short.valueOf(t);
row.set(c, s);
break;
case INT:
Integer i;
i = Integer.valueOf(t);
row.set(c, i);
break;
case LONG:
Long l;
l = Long.valueOf(t);
row.set(c, l);
break;
case FLOAT:
Float f;
f = Float.valueOf(t);
row.set(c, f);
break;
case DOUBLE:
Double d;
d = Double.valueOf(t);
row.set(c, d);
break;
case BOOLEAN:
Boolean bool;
bool = Boolean.valueOf(t);
row.set(c, bool);
break;
case TIMESTAMP:
Timestamp ts;
ts = Timestamp.valueOf(t);
row.set(c, ts);
break;
case DATE:
Date date;
date = Date.valueOf(t);
row.set(c, date);
break;
case DECIMAL:
HiveDecimal bd = HiveDecimal.create(t);
row.set(c, bd);
break;
case CHAR:
HiveChar hc = new HiveChar(t, ((CharTypeInfo) typeInfo).getLength());
row.set(c, hc);
break;
case VARCHAR:
HiveVarchar hv = new HiveVarchar(t, ((VarcharTypeInfo) typeInfo).getLength());
row.set(c, hv);
break;
default:
throw new SerDeException("Unsupported type " + typeInfo);
}
} catch (RuntimeException e) {
partialMatchedRowsCount++;
if (!alreadyLoggedPartialMatch) {
// Report the row if its the first row
log.warn("" + partialMatchedRowsCount + " partially unmatched rows are found, " + " cannot find group " + c + ": " + rowText);
alreadyLoggedPartialMatch = true;
}
row.set(c, null);
}
}
return row;
}
use of org.apache.hadoop.hive.common.type.Date in project hive by apache.
the class VerifyFast method doVerifyDeserializeRead.
public static void doVerifyDeserializeRead(DeserializeRead deserializeRead, TypeInfo typeInfo, Object object, boolean isNull) throws IOException {
if (isNull) {
if (object != null) {
TestCase.fail("Field reports null but object is not null (class " + object.getClass().getName() + ", " + object.toString() + ")");
}
return;
} else if (object == null) {
TestCase.fail("Field report not null but object is null");
}
switch(typeInfo.getCategory()) {
case PRIMITIVE:
{
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
switch(primitiveTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
{
boolean value = deserializeRead.currentBoolean;
if (!(object instanceof BooleanWritable)) {
TestCase.fail("Boolean expected writable not Boolean");
}
boolean expected = ((BooleanWritable) object).get();
if (value != expected) {
TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case BYTE:
{
byte value = deserializeRead.currentByte;
if (!(object instanceof ByteWritable)) {
TestCase.fail("Byte expected writable not Byte");
}
byte expected = ((ByteWritable) object).get();
if (value != expected) {
TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
}
}
break;
case SHORT:
{
short value = deserializeRead.currentShort;
if (!(object instanceof ShortWritable)) {
TestCase.fail("Short expected writable not Short");
}
short expected = ((ShortWritable) object).get();
if (value != expected) {
TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case INT:
{
int value = deserializeRead.currentInt;
if (!(object instanceof IntWritable)) {
TestCase.fail("Integer expected writable not Integer");
}
int expected = ((IntWritable) object).get();
if (value != expected) {
TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case LONG:
{
long value = deserializeRead.currentLong;
if (!(object instanceof LongWritable)) {
TestCase.fail("Long expected writable not Long");
}
Long expected = ((LongWritable) object).get();
if (value != expected) {
TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case FLOAT:
{
float value = deserializeRead.currentFloat;
if (!(object instanceof FloatWritable)) {
TestCase.fail("Float expected writable not Float");
}
float expected = ((FloatWritable) object).get();
if (value != expected) {
TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case DOUBLE:
{
double value = deserializeRead.currentDouble;
if (!(object instanceof DoubleWritable)) {
TestCase.fail("Double expected writable not Double");
}
double expected = ((DoubleWritable) object).get();
if (value != expected) {
TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case STRING:
{
byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
Text text = new Text(stringBytes);
String string = text.toString();
String expected = ((Text) object).toString();
if (!string.equals(expected)) {
TestCase.fail("String field mismatch (expected '" + expected + "' found '" + string + "')");
}
}
break;
case CHAR:
{
byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
Text text = new Text(stringBytes);
String string = text.toString();
HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
HiveChar expected = ((HiveCharWritable) object).getHiveChar();
if (!hiveChar.equals(expected)) {
TestCase.fail("Char field mismatch (expected '" + expected + "' found '" + hiveChar + "')");
}
}
break;
case VARCHAR:
{
byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
Text text = new Text(stringBytes);
String string = text.toString();
HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
HiveVarchar expected = ((HiveVarcharWritable) object).getHiveVarchar();
if (!hiveVarchar.equals(expected)) {
TestCase.fail("Varchar field mismatch (expected '" + expected + "' found '" + hiveVarchar + "')");
}
}
break;
case DECIMAL:
{
HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
if (value == null) {
TestCase.fail("Decimal field evaluated to NULL");
}
HiveDecimal expected = ((HiveDecimalWritable) object).getHiveDecimal();
if (!value.equals(expected)) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
int precision = decimalTypeInfo.getPrecision();
int scale = decimalTypeInfo.getScale();
TestCase.fail("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
}
}
break;
case DATE:
{
Date value = deserializeRead.currentDateWritable.get();
Date expected = ((DateWritableV2) object).get();
if (!value.equals(expected)) {
TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case TIMESTAMP:
{
Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
Timestamp expected = ((TimestampWritableV2) object).getTimestamp();
if (!value.equals(expected)) {
TestCase.fail("Timestamp field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case INTERVAL_YEAR_MONTH:
{
HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
if (!value.equals(expected)) {
TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case INTERVAL_DAY_TIME:
{
HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
if (!value.equals(expected)) {
TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case BINARY:
{
byte[] byteArray = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
BytesWritable bytesWritable = (BytesWritable) object;
byte[] expected = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
if (byteArray.length != expected.length) {
TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
}
for (int b = 0; b < byteArray.length; b++) {
if (byteArray[b] != expected[b]) {
TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
}
}
}
break;
default:
throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
}
}
break;
case LIST:
case MAP:
case STRUCT:
case UNION:
throw new Error("Complex types need to be handled separately");
default:
throw new Error("Unknown category " + typeInfo.getCategory());
}
}
Aggregations