use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class VectorVerifyFast method doVerifyDeserializeRead.
public static void doVerifyDeserializeRead(DeserializeRead deserializeRead, TypeInfo typeInfo, Object object, boolean isNull) throws IOException {
if (isNull) {
if (object != null) {
TestCase.fail("Field reports null but object is not null (class " + object.getClass().getName() + ", " + object.toString() + ")");
}
return;
} else if (object == null) {
TestCase.fail("Field report not null but object is null");
}
switch(typeInfo.getCategory()) {
case PRIMITIVE:
{
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
switch(primitiveTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
{
boolean value = deserializeRead.currentBoolean;
if (!(object instanceof BooleanWritable)) {
TestCase.fail("Boolean expected writable not Boolean");
}
boolean expected = ((BooleanWritable) object).get();
if (value != expected) {
TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case BYTE:
{
byte value = deserializeRead.currentByte;
if (!(object instanceof ByteWritable)) {
TestCase.fail("Byte expected writable not Byte");
}
byte expected = ((ByteWritable) object).get();
if (value != expected) {
TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
}
}
break;
case SHORT:
{
short value = deserializeRead.currentShort;
if (!(object instanceof ShortWritable)) {
TestCase.fail("Short expected writable not Short");
}
short expected = ((ShortWritable) object).get();
if (value != expected) {
TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case INT:
{
int value = deserializeRead.currentInt;
if (!(object instanceof IntWritable)) {
TestCase.fail("Integer expected writable not Integer");
}
int expected = ((IntWritable) object).get();
if (value != expected) {
TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case LONG:
{
long value = deserializeRead.currentLong;
if (!(object instanceof LongWritable)) {
TestCase.fail("Long expected writable not Long");
}
Long expected = ((LongWritable) object).get();
if (value != expected) {
TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case FLOAT:
{
float value = deserializeRead.currentFloat;
if (!(object instanceof FloatWritable)) {
TestCase.fail("Float expected writable not Float");
}
float expected = ((FloatWritable) object).get();
if (value != expected) {
TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case DOUBLE:
{
double value = deserializeRead.currentDouble;
if (!(object instanceof DoubleWritable)) {
TestCase.fail("Double expected writable not Double");
}
double expected = ((DoubleWritable) object).get();
if (value != expected) {
TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case STRING:
{
byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
Text text = new Text(stringBytes);
String string = text.toString();
String expected = ((Text) object).toString();
if (!string.equals(expected)) {
TestCase.fail("String field mismatch (expected '" + expected + "' found '" + string + "')");
}
}
break;
case CHAR:
{
byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
Text text = new Text(stringBytes);
String string = text.toString();
HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
HiveChar expected = ((HiveCharWritable) object).getHiveChar();
if (!hiveChar.equals(expected)) {
TestCase.fail("Char field mismatch (expected '" + expected + "' found '" + hiveChar + "')");
}
}
break;
case VARCHAR:
{
byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
Text text = new Text(stringBytes);
String string = text.toString();
HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
HiveVarchar expected = ((HiveVarcharWritable) object).getHiveVarchar();
if (!hiveVarchar.equals(expected)) {
TestCase.fail("Varchar field mismatch (expected '" + expected + "' found '" + hiveVarchar + "')");
}
}
break;
case DECIMAL:
{
HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
if (value == null) {
TestCase.fail("Decimal field evaluated to NULL");
}
HiveDecimal expected = ((HiveDecimalWritable) object).getHiveDecimal();
if (!value.equals(expected)) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
int precision = decimalTypeInfo.getPrecision();
int scale = decimalTypeInfo.getScale();
TestCase.fail("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
}
}
break;
case DATE:
{
Date value = deserializeRead.currentDateWritable.get();
Date expected = ((DateWritable) object).get();
if (!value.equals(expected)) {
TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case TIMESTAMP:
{
Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
Timestamp expected = ((TimestampWritable) object).getTimestamp();
if (!value.equals(expected)) {
TestCase.fail("Timestamp field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case INTERVAL_YEAR_MONTH:
{
HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
if (!value.equals(expected)) {
TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case INTERVAL_DAY_TIME:
{
HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
if (!value.equals(expected)) {
TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
}
}
break;
case BINARY:
{
byte[] byteArray = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
BytesWritable bytesWritable = (BytesWritable) object;
byte[] expected = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
if (byteArray.length != expected.length) {
TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
}
for (int b = 0; b < byteArray.length; b++) {
if (byteArray[b] != expected[b]) {
TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(byteArray) + ")");
}
}
}
break;
default:
throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
}
}
break;
case LIST:
case MAP:
case STRUCT:
case UNION:
throw new Error("Complex types need to be handled separately");
default:
throw new Error("Unknown category " + typeInfo.getCategory());
}
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class BinarySortableSerDe method deserialize.
static Object deserialize(InputByteBuffer buffer, TypeInfo type, boolean invert, byte nullMarker, byte notNullMarker, Object reuse) throws IOException {
// Is this field a null?
byte isNull = buffer.read(invert);
if (isNull == nullMarker) {
return null;
}
assert (isNull == notNullMarker);
switch(type.getCategory()) {
case PRIMITIVE:
{
PrimitiveTypeInfo ptype = (PrimitiveTypeInfo) type;
switch(ptype.getPrimitiveCategory()) {
case VOID:
{
return null;
}
case BOOLEAN:
{
BooleanWritable r = reuse == null ? new BooleanWritable() : (BooleanWritable) reuse;
byte b = buffer.read(invert);
assert (b == 1 || b == 2);
r.set(b == 2);
return r;
}
case BYTE:
{
ByteWritable r = reuse == null ? new ByteWritable() : (ByteWritable) reuse;
r.set((byte) (buffer.read(invert) ^ 0x80));
return r;
}
case SHORT:
{
ShortWritable r = reuse == null ? new ShortWritable() : (ShortWritable) reuse;
int v = buffer.read(invert) ^ 0x80;
v = (v << 8) + (buffer.read(invert) & 0xff);
r.set((short) v);
return r;
}
case INT:
{
IntWritable r = reuse == null ? new IntWritable() : (IntWritable) reuse;
r.set(deserializeInt(buffer, invert));
return r;
}
case LONG:
{
LongWritable r = reuse == null ? new LongWritable() : (LongWritable) reuse;
r.set(deserializeLong(buffer, invert));
return r;
}
case FLOAT:
{
FloatWritable r = reuse == null ? new FloatWritable() : (FloatWritable) reuse;
int v = 0;
for (int i = 0; i < 4; i++) {
v = (v << 8) + (buffer.read(invert) & 0xff);
}
if ((v & (1 << 31)) == 0) {
// negative number, flip all bits
v = ~v;
} else {
// positive number, flip the first bit
v = v ^ (1 << 31);
}
r.set(Float.intBitsToFloat(v));
return r;
}
case DOUBLE:
{
DoubleWritable r = reuse == null ? new DoubleWritable() : (DoubleWritable) reuse;
long v = 0;
for (int i = 0; i < 8; i++) {
v = (v << 8) + (buffer.read(invert) & 0xff);
}
if ((v & (1L << 63)) == 0) {
// negative number, flip all bits
v = ~v;
} else {
// positive number, flip the first bit
v = v ^ (1L << 63);
}
r.set(Double.longBitsToDouble(v));
return r;
}
case STRING:
{
Text r = reuse == null ? new Text() : (Text) reuse;
return deserializeText(buffer, invert, r);
}
case CHAR:
{
HiveCharWritable r = reuse == null ? new HiveCharWritable() : (HiveCharWritable) reuse;
// Use internal text member to read value
deserializeText(buffer, invert, r.getTextValue());
r.enforceMaxLength(getCharacterMaxLength(type));
return r;
}
case VARCHAR:
{
HiveVarcharWritable r = reuse == null ? new HiveVarcharWritable() : (HiveVarcharWritable) reuse;
// Use HiveVarchar's internal Text member to read the value.
deserializeText(buffer, invert, r.getTextValue());
// If we cache helper data for deserialization we could avoid having
// to call getVarcharMaxLength() on every deserialize call.
r.enforceMaxLength(getCharacterMaxLength(type));
return r;
}
case BINARY:
{
BytesWritable bw = new BytesWritable();
// Get the actual length first
int start = buffer.tell();
int length = 0;
do {
byte b = buffer.read(invert);
if (b == 0) {
// end of string
break;
}
if (b == 1) {
// the last char is an escape char. read the actual char
buffer.read(invert);
}
length++;
} while (true);
if (length == buffer.tell() - start) {
// No escaping happened, so we are already done.
bw.set(buffer.getData(), start, length);
} else {
// Escaping happened, we need to copy byte-by-byte.
// 1. Set the length first.
bw.set(buffer.getData(), start, length);
// 2. Reset the pointer.
buffer.seek(start);
// 3. Copy the data.
byte[] rdata = bw.getBytes();
for (int i = 0; i < length; i++) {
byte b = buffer.read(invert);
if (b == 1) {
// The last char is an escape char, read the actual char.
// The serialization format escape \0 to \1, and \1 to \2,
// to make sure the string is null-terminated.
b = (byte) (buffer.read(invert) - 1);
}
rdata[i] = b;
}
// 4. Read the null terminator.
byte b = buffer.read(invert);
assert (b == 0);
}
return bw;
}
case DATE:
{
DateWritable d = reuse == null ? new DateWritable() : (DateWritable) reuse;
d.set(deserializeInt(buffer, invert));
return d;
}
case TIMESTAMP:
TimestampWritable t = (reuse == null ? new TimestampWritable() : (TimestampWritable) reuse);
byte[] bytes = new byte[TimestampWritable.BINARY_SORTABLE_LENGTH];
for (int i = 0; i < bytes.length; i++) {
bytes[i] = buffer.read(invert);
}
t.setBinarySortable(bytes, 0);
return t;
case TIMESTAMPLOCALTZ:
TimestampLocalTZWritable tstz = (reuse == null ? new TimestampLocalTZWritable() : (TimestampLocalTZWritable) reuse);
byte[] data = new byte[TimestampLocalTZWritable.BINARY_SORTABLE_LENGTH];
for (int i = 0; i < data.length; i++) {
data[i] = buffer.read(invert);
}
// Across MR process boundary tz is normalized and stored in type
// and is not carried in data for each row.
tstz.fromBinarySortable(data, 0, ((TimestampLocalTZTypeInfo) type).timeZone());
return tstz;
case INTERVAL_YEAR_MONTH:
{
HiveIntervalYearMonthWritable i = reuse == null ? new HiveIntervalYearMonthWritable() : (HiveIntervalYearMonthWritable) reuse;
i.set(deserializeInt(buffer, invert));
return i;
}
case INTERVAL_DAY_TIME:
{
HiveIntervalDayTimeWritable i = reuse == null ? new HiveIntervalDayTimeWritable() : (HiveIntervalDayTimeWritable) reuse;
long totalSecs = deserializeLong(buffer, invert);
int nanos = deserializeInt(buffer, invert);
i.set(totalSecs, nanos);
return i;
}
case DECIMAL:
{
// See serialization of decimal for explanation (below)
HiveDecimalWritable bdw = (reuse == null ? new HiveDecimalWritable() : (HiveDecimalWritable) reuse);
int b = buffer.read(invert) - 1;
assert (b == 1 || b == -1 || b == 0);
boolean positive = b != -1;
int factor = buffer.read(invert) ^ 0x80;
for (int i = 0; i < 3; i++) {
factor = (factor << 8) + (buffer.read(invert) & 0xff);
}
if (!positive) {
factor = -factor;
}
int start = buffer.tell();
int length = 0;
do {
b = buffer.read(positive ? invert : !invert);
assert (b != 1);
if (b == 0) {
// end of digits
break;
}
length++;
} while (true);
final byte[] decimalBuffer = new byte[length];
buffer.seek(start);
for (int i = 0; i < length; ++i) {
decimalBuffer[i] = buffer.read(positive ? invert : !invert);
}
// read the null byte again
buffer.read(positive ? invert : !invert);
String digits = new String(decimalBuffer, 0, length, decimalCharSet);
BigInteger bi = new BigInteger(digits);
HiveDecimal bd = HiveDecimal.create(bi).scaleByPowerOfTen(factor - length);
if (!positive) {
bd = bd.negate();
}
bdw.set(bd);
return bdw;
}
default:
{
throw new RuntimeException("Unrecognized type: " + ptype.getPrimitiveCategory());
}
}
}
case LIST:
{
ListTypeInfo ltype = (ListTypeInfo) type;
TypeInfo etype = ltype.getListElementTypeInfo();
// Create the list if needed
ArrayList<Object> r = reuse == null ? new ArrayList<Object>() : (ArrayList<Object>) reuse;
// Read the list
int size = 0;
while (true) {
int more = buffer.read(invert);
if (more == 0) {
// \0 to terminate
break;
}
// \1 followed by each element
assert (more == 1);
if (size == r.size()) {
r.add(null);
}
r.set(size, deserialize(buffer, etype, invert, nullMarker, notNullMarker, r.get(size)));
size++;
}
// Remove additional elements if the list is reused
while (r.size() > size) {
r.remove(r.size() - 1);
}
return r;
}
case MAP:
{
MapTypeInfo mtype = (MapTypeInfo) type;
TypeInfo ktype = mtype.getMapKeyTypeInfo();
TypeInfo vtype = mtype.getMapValueTypeInfo();
// Create the map if needed
Map<Object, Object> r;
if (reuse == null) {
r = new HashMap<Object, Object>();
} else {
r = (HashMap<Object, Object>) reuse;
r.clear();
}
while (true) {
int more = buffer.read(invert);
if (more == 0) {
// \0 to terminate
break;
}
// \1 followed by each key and then each value
assert (more == 1);
Object k = deserialize(buffer, ktype, invert, nullMarker, notNullMarker, null);
Object v = deserialize(buffer, vtype, invert, nullMarker, notNullMarker, null);
r.put(k, v);
}
return r;
}
case STRUCT:
{
StructTypeInfo stype = (StructTypeInfo) type;
List<TypeInfo> fieldTypes = stype.getAllStructFieldTypeInfos();
int size = fieldTypes.size();
// Create the struct if needed
ArrayList<Object> r = reuse == null ? new ArrayList<Object>(size) : (ArrayList<Object>) reuse;
assert (r.size() <= size);
// Set the size of the struct
while (r.size() < size) {
r.add(null);
}
// Read one field by one field
for (int eid = 0; eid < size; eid++) {
r.set(eid, deserialize(buffer, fieldTypes.get(eid), invert, nullMarker, notNullMarker, r.get(eid)));
}
return r;
}
case UNION:
{
UnionTypeInfo utype = (UnionTypeInfo) type;
StandardUnion r = reuse == null ? new StandardUnion() : (StandardUnion) reuse;
// Read the tag
byte tag = buffer.read(invert);
r.setTag(tag);
r.setObject(deserialize(buffer, utype.getAllUnionObjectTypeInfos().get(tag), invert, nullMarker, notNullMarker, null));
return r;
}
default:
{
throw new RuntimeException("Unrecognized type: " + type.getCategory());
}
}
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class VerifyFast method serializeWrite.
public static void serializeWrite(SerializeWrite serializeWrite, TypeInfo typeInfo, Object object) throws IOException {
if (object == null) {
serializeWrite.writeNull();
return;
}
switch(typeInfo.getCategory()) {
case PRIMITIVE:
{
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
switch(primitiveTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
{
boolean value = ((BooleanWritable) object).get();
serializeWrite.writeBoolean(value);
}
break;
case BYTE:
{
byte value = ((ByteWritable) object).get();
serializeWrite.writeByte(value);
}
break;
case SHORT:
{
short value = ((ShortWritable) object).get();
serializeWrite.writeShort(value);
}
break;
case INT:
{
int value = ((IntWritable) object).get();
serializeWrite.writeInt(value);
}
break;
case LONG:
{
long value = ((LongWritable) object).get();
serializeWrite.writeLong(value);
}
break;
case FLOAT:
{
float value = ((FloatWritable) object).get();
serializeWrite.writeFloat(value);
}
break;
case DOUBLE:
{
double value = ((DoubleWritable) object).get();
serializeWrite.writeDouble(value);
}
break;
case STRING:
{
Text value = (Text) object;
byte[] stringBytes = value.getBytes();
int stringLength = stringBytes.length;
serializeWrite.writeString(stringBytes, 0, stringLength);
}
break;
case CHAR:
{
HiveChar value = ((HiveCharWritable) object).getHiveChar();
serializeWrite.writeHiveChar(value);
}
break;
case VARCHAR:
{
HiveVarchar value = ((HiveVarcharWritable) object).getHiveVarchar();
serializeWrite.writeHiveVarchar(value);
}
break;
case DECIMAL:
{
HiveDecimal value = ((HiveDecimalWritable) object).getHiveDecimal();
DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
serializeWrite.writeHiveDecimal(value, decTypeInfo.scale());
}
break;
case DATE:
{
Date value = ((DateWritable) object).get();
serializeWrite.writeDate(value);
}
break;
case TIMESTAMP:
{
Timestamp value = ((TimestampWritable) object).getTimestamp();
serializeWrite.writeTimestamp(value);
}
break;
case INTERVAL_YEAR_MONTH:
{
HiveIntervalYearMonth value = ((HiveIntervalYearMonthWritable) object).getHiveIntervalYearMonth();
serializeWrite.writeHiveIntervalYearMonth(value);
}
break;
case INTERVAL_DAY_TIME:
{
HiveIntervalDayTime value = ((HiveIntervalDayTimeWritable) object).getHiveIntervalDayTime();
serializeWrite.writeHiveIntervalDayTime(value);
}
break;
case BINARY:
{
BytesWritable byteWritable = (BytesWritable) object;
byte[] binaryBytes = byteWritable.getBytes();
int length = byteWritable.getLength();
serializeWrite.writeBinary(binaryBytes, 0, length);
}
break;
default:
throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory().name());
}
}
break;
case LIST:
{
ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
TypeInfo elementTypeInfo = listTypeInfo.getListElementTypeInfo();
ArrayList<Object> elements = (ArrayList<Object>) object;
serializeWrite.beginList(elements);
boolean isFirst = true;
for (Object elementObject : elements) {
if (isFirst) {
isFirst = false;
} else {
serializeWrite.separateList();
}
if (elementObject == null) {
serializeWrite.writeNull();
} else {
serializeWrite(serializeWrite, elementTypeInfo, elementObject);
}
}
serializeWrite.finishList();
}
break;
case MAP:
{
MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
TypeInfo keyTypeInfo = mapTypeInfo.getMapKeyTypeInfo();
TypeInfo valueTypeInfo = mapTypeInfo.getMapValueTypeInfo();
HashMap<Object, Object> hashMap = (HashMap<Object, Object>) object;
serializeWrite.beginMap(hashMap);
boolean isFirst = true;
for (Entry<Object, Object> entry : hashMap.entrySet()) {
if (isFirst) {
isFirst = false;
} else {
serializeWrite.separateKeyValuePair();
}
if (entry.getKey() == null) {
serializeWrite.writeNull();
} else {
serializeWrite(serializeWrite, keyTypeInfo, entry.getKey());
}
serializeWrite.separateKey();
if (entry.getValue() == null) {
serializeWrite.writeNull();
} else {
serializeWrite(serializeWrite, valueTypeInfo, entry.getValue());
}
}
serializeWrite.finishMap();
}
break;
case STRUCT:
{
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
ArrayList<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
ArrayList<Object> fieldValues = (ArrayList<Object>) object;
final int size = fieldValues.size();
serializeWrite.beginStruct(fieldValues);
boolean isFirst = true;
for (int i = 0; i < size; i++) {
if (isFirst) {
isFirst = false;
} else {
serializeWrite.separateStruct();
}
serializeWrite(serializeWrite, fieldTypeInfos.get(i), fieldValues.get(i));
}
serializeWrite.finishStruct();
}
break;
case UNION:
{
UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
List<TypeInfo> fieldTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
final int size = fieldTypeInfos.size();
StandardUnion standardUnion = (StandardUnion) object;
byte tag = standardUnion.getTag();
serializeWrite.beginUnion(tag);
serializeWrite(serializeWrite, fieldTypeInfos.get(tag), standardUnion.getObject());
serializeWrite.finishUnion();
}
break;
default:
throw new Error("Unknown category " + typeInfo.getCategory().name());
}
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class VerifyLazy method lazyCompare.
public static boolean lazyCompare(TypeInfo typeInfo, Object lazyObject, Object expectedObject) {
if (expectedObject == null) {
if (lazyObject != null) {
throw new RuntimeException("Expected object is null but object is not null " + lazyObject.toString() + " typeInfo " + typeInfo.toString());
}
return true;
} else if (lazyObject == null) {
throw new RuntimeException("Expected object is not null \"" + expectedObject.toString() + "\" typeInfo " + typeInfo.toString() + " but object is null");
}
if (lazyObject instanceof Writable) {
if (!lazyObject.equals(expectedObject)) {
throw new RuntimeException("Expected object " + expectedObject.toString() + " and actual object " + lazyObject.toString() + " is not equal typeInfo " + typeInfo.toString());
}
return true;
}
if (lazyObject instanceof LazyPrimitive) {
Object primitiveObject = ((LazyPrimitive) lazyObject).getObject();
PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
switch(primitiveTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
{
if (!(primitiveObject instanceof LazyBoolean)) {
throw new RuntimeException("Expected LazyBoolean");
}
boolean value = ((LazyBoolean) primitiveObject).getWritableObject().get();
boolean expected = ((BooleanWritable) expectedObject).get();
if (value != expected) {
throw new RuntimeException("Boolean field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case BYTE:
{
if (!(primitiveObject instanceof LazyByte)) {
throw new RuntimeException("Expected LazyByte");
}
byte value = ((LazyByte) primitiveObject).getWritableObject().get();
byte expected = ((ByteWritable) expectedObject).get();
if (value != expected) {
throw new RuntimeException("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
}
}
break;
case SHORT:
{
if (!(primitiveObject instanceof LazyShort)) {
throw new RuntimeException("Expected LazyShort");
}
short value = ((LazyShort) primitiveObject).getWritableObject().get();
short expected = ((ShortWritable) expectedObject).get();
if (value != expected) {
throw new RuntimeException("Short field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case INT:
{
if (!(primitiveObject instanceof LazyInteger)) {
throw new RuntimeException("Expected LazyInteger");
}
int value = ((LazyInteger) primitiveObject).getWritableObject().get();
int expected = ((IntWritable) expectedObject).get();
if (value != expected) {
throw new RuntimeException("Int field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case LONG:
{
if (!(primitiveObject instanceof LazyLong)) {
throw new RuntimeException("Expected LazyLong");
}
long value = ((LazyLong) primitiveObject).getWritableObject().get();
long expected = ((LongWritable) expectedObject).get();
if (value != expected) {
throw new RuntimeException("Long field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case FLOAT:
{
if (!(primitiveObject instanceof LazyFloat)) {
throw new RuntimeException("Expected LazyFloat");
}
float value = ((LazyFloat) primitiveObject).getWritableObject().get();
float expected = ((FloatWritable) expectedObject).get();
if (value != expected) {
throw new RuntimeException("Float field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case DOUBLE:
{
if (!(primitiveObject instanceof LazyDouble)) {
throw new RuntimeException("Expected LazyDouble");
}
double value = ((LazyDouble) primitiveObject).getWritableObject().get();
double expected = ((DoubleWritable) expectedObject).get();
if (value != expected) {
throw new RuntimeException("Double field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case STRING:
{
if (!(primitiveObject instanceof LazyString)) {
throw new RuntimeException("Text expected writable not Text");
}
Text value = ((LazyString) primitiveObject).getWritableObject();
Text expected = ((Text) expectedObject);
if (!value.equals(expected)) {
throw new RuntimeException("String field mismatch (expected '" + expected + "' found '" + value + "')");
}
}
break;
case CHAR:
{
if (!(primitiveObject instanceof LazyHiveChar)) {
throw new RuntimeException("Expected LazyHiveChar");
}
HiveChar value = ((LazyHiveChar) primitiveObject).getWritableObject().getHiveChar();
HiveChar expected = ((HiveCharWritable) expectedObject).getHiveChar();
if (!value.equals(expected)) {
throw new RuntimeException("HiveChar field mismatch (expected '" + expected + "' found '" + value + "')");
}
}
break;
case VARCHAR:
{
if (!(primitiveObject instanceof LazyHiveVarchar)) {
throw new RuntimeException("Expected LazyHiveVarchar");
}
HiveVarchar value = ((LazyHiveVarchar) primitiveObject).getWritableObject().getHiveVarchar();
HiveVarchar expected = ((HiveVarcharWritable) expectedObject).getHiveVarchar();
if (!value.equals(expected)) {
throw new RuntimeException("HiveVarchar field mismatch (expected '" + expected + "' found '" + value + "')");
}
}
break;
case DECIMAL:
{
if (!(primitiveObject instanceof LazyHiveDecimal)) {
throw new RuntimeException("Expected LazyDecimal");
}
HiveDecimal value = ((LazyHiveDecimal) primitiveObject).getWritableObject().getHiveDecimal();
HiveDecimal expected = ((HiveDecimalWritable) expectedObject).getHiveDecimal();
if (!value.equals(expected)) {
DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
int precision = decimalTypeInfo.getPrecision();
int scale = decimalTypeInfo.getScale();
throw new RuntimeException("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
}
}
break;
case DATE:
{
if (!(primitiveObject instanceof LazyDate)) {
throw new RuntimeException("Expected LazyDate");
}
Date value = ((LazyDate) primitiveObject).getWritableObject().get();
Date expected = ((DateWritable) expectedObject).get();
if (!value.equals(expected)) {
throw new RuntimeException("Date field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case TIMESTAMP:
{
if (!(primitiveObject instanceof LazyTimestamp)) {
throw new RuntimeException("TimestampWritable expected writable not TimestampWritable");
}
Timestamp value = ((LazyTimestamp) primitiveObject).getWritableObject().getTimestamp();
Timestamp expected = ((TimestampWritable) expectedObject).getTimestamp();
if (!value.equals(expected)) {
throw new RuntimeException("Timestamp field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case INTERVAL_YEAR_MONTH:
{
if (!(primitiveObject instanceof LazyHiveIntervalYearMonth)) {
throw new RuntimeException("Expected LazyHiveIntervalYearMonth");
}
HiveIntervalYearMonth value = ((LazyHiveIntervalYearMonth) primitiveObject).getWritableObject().getHiveIntervalYearMonth();
HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) expectedObject).getHiveIntervalYearMonth();
if (!value.equals(expected)) {
throw new RuntimeException("HiveIntervalYearMonth field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case INTERVAL_DAY_TIME:
{
if (!(primitiveObject instanceof LazyHiveIntervalDayTime)) {
throw new RuntimeException("Expected writable LazyHiveIntervalDayTime");
}
HiveIntervalDayTime value = ((LazyHiveIntervalDayTime) primitiveObject).getWritableObject().getHiveIntervalDayTime();
HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) expectedObject).getHiveIntervalDayTime();
if (!value.equals(expected)) {
throw new RuntimeException("HiveIntervalDayTime field mismatch (expected " + expected + " found " + value + ")");
}
}
break;
case BINARY:
{
if (!(primitiveObject instanceof LazyBinary)) {
throw new RuntimeException("Expected LazyBinary");
}
BytesWritable bytesWritable = ((LazyBinary) primitiveObject).getWritableObject();
byte[] value = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
BytesWritable bytesWritableExpected = (BytesWritable) expectedObject;
byte[] expected = Arrays.copyOfRange(bytesWritableExpected.getBytes(), 0, bytesWritableExpected.getLength());
if (value.length != expected.length) {
throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
}
for (int b = 0; b < value.length; b++) {
if (value[b] != expected[b]) {
throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
}
}
}
break;
default:
throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
}
} else if (lazyObject instanceof LazyArray) {
LazyArray lazyArray = (LazyArray) lazyObject;
List<Object> list = lazyArray.getList();
List<Object> expectedList = (List<Object>) expectedObject;
ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
if (list.size() != expectedList.size()) {
throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")" + " elementTypeInfo " + listTypeInfo.getListElementTypeInfo().toString());
}
return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
} else if (typeInfo instanceof ListTypeInfo) {
List<Object> list;
if (lazyObject instanceof LazyBinaryArray) {
list = ((LazyBinaryArray) lazyObject).getList();
} else {
list = (List<Object>) lazyObject;
}
List<Object> expectedList = (List<Object>) expectedObject;
if (list.size() != expectedList.size()) {
throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")");
}
return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
} else if (lazyObject instanceof LazyMap) {
LazyMap lazyMap = (LazyMap) lazyObject;
Map<Object, Object> map = lazyMap.getMap();
Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
} else if (typeInfo instanceof MapTypeInfo) {
Map<Object, Object> map;
Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
if (lazyObject instanceof LazyBinaryMap) {
map = ((LazyBinaryMap) lazyObject).getMap();
} else {
map = (Map<Object, Object>) lazyObject;
}
return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
} else if (lazyObject instanceof LazyStruct) {
LazyStruct lazyStruct = (LazyStruct) lazyObject;
List<Object> fields = lazyStruct.getFieldsAsList();
List<Object> expectedFields = (List<Object>) expectedObject;
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
return lazyCompareStruct(structTypeInfo, fields, expectedFields);
} else if (typeInfo instanceof StructTypeInfo) {
ArrayList<Object> fields;
if (lazyObject instanceof LazyBinaryStruct) {
fields = ((LazyBinaryStruct) lazyObject).getFieldsAsList();
} else {
fields = (ArrayList<Object>) lazyObject;
}
List<Object> expectedFields = (List<Object>) expectedObject;
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
return lazyCompareStruct(structTypeInfo, fields, expectedFields);
} else if (lazyObject instanceof LazyUnion) {
LazyUnion union = (LazyUnion) lazyObject;
StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
return lazyCompareUnion(unionTypeInfo, union, expectedUnion);
} else if (typeInfo instanceof UnionTypeInfo) {
StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
if (lazyObject instanceof LazyBinaryUnion) {
return lazyCompareUnion(unionTypeInfo, (LazyBinaryUnion) lazyObject, expectedUnion);
} else {
return lazyCompareUnion(unionTypeInfo, (UnionObject) lazyObject, expectedUnion);
}
} else {
System.err.println("Not implemented " + typeInfo.getClass().getName());
}
return true;
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class TestLazySimpleSerDe method testLazySimpleSerDeLastColumnTakesRest.
/**
* Test the LazySimpleSerDe class with LastColumnTakesRest option.
*/
public void testLazySimpleSerDeLastColumnTakesRest() throws Throwable {
try {
// Create the SerDe
LazySimpleSerDe serDe = new LazySimpleSerDe();
Configuration conf = new Configuration();
Properties tbl = createProperties();
tbl.setProperty(serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST, "true");
SerDeUtils.initializeSerDe(serDe, conf, tbl, null);
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\ta\tb\t");
String s = "123\t456\t789\t1000\t5.3\thive and hadoop\t1\ta\tb\t";
Object[] expectedFieldsData = { new ByteWritable((byte) 123), new ShortWritable((short) 456), new IntWritable(789), new LongWritable(1000), new DoubleWritable(5.3), new Text("hive and hadoop"), new IntWritable(1), new Text("a\tb\t") };
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
Aggregations