use of org.apache.hadoop.hive.serde2.SerDeException in project hive by apache.
the class TestAvroDeserializer method canDeserializeEnums.
// Enums are one of two types we fudge for Hive. Enums go in, Strings come out.
@Test
public void canDeserializeEnums() throws SerDeException, IOException {
Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.ENUM_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
record.put("baddies", new GenericData.EnumSymbol(s.getField("baddies").schema(), "DALEKS"));
assertTrue(GENERIC_DATA.validate(s, record));
AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
AvroDeserializer de = new AvroDeserializer();
ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
assertEquals(1, row.size());
StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
assertEquals(1, fieldRefs.size());
StructField fieldRef = fieldRefs.get(0);
assertEquals("baddies", fieldRef.getFieldName());
Object theStringObject = oi.getStructFieldData(row, fieldRef);
assertTrue(fieldRef.getFieldObjectInspector() instanceof StringObjectInspector);
StringObjectInspector soi = (StringObjectInspector) fieldRef.getFieldObjectInspector();
String finalValue = soi.getPrimitiveJavaObject(theStringObject);
assertEquals("DALEKS", finalValue);
}
use of org.apache.hadoop.hive.serde2.SerDeException in project hive by apache.
the class TestAvroDeserializer method canDeserializeMapsWithJavaLangStringKeys.
@Test
public void canDeserializeMapsWithJavaLangStringKeys() throws IOException, SerDeException {
// Ensures maps can be deserialized when avro.java.string=String.
// See http://stackoverflow.com/a/19868919/312944 for why that might be used.
String schemaString = "{\n" + " \"namespace\": \"testing\",\n" + " \"name\": \"oneMap\",\n" + " \"type\": \"record\",\n" + " \"fields\": [\n" + " {\n" + " \"name\":\"aMap\",\n" + " \"type\":{\"type\":\"map\",\n" + " \"avro.java.string\":\"String\",\n" + " \"values\":\"long\"}\n" + "\t}\n" + " ]\n" + "}";
Schema s = AvroSerdeUtils.getSchemaFor(schemaString);
GenericData.Record record = new GenericData.Record(s);
Map<String, Long> m = new Hashtable<String, Long>();
m.put("one", 1l);
m.put("two", 2l);
m.put("three", 3l);
record.put("aMap", m);
assertTrue(GENERIC_DATA.validate(s, record));
System.out.println("record = " + record);
AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
AvroDeserializer de = new AvroDeserializer();
ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
assertEquals(1, row.size());
Object theMapObject = row.get(0);
assertTrue(theMapObject instanceof Map);
Map theMap = (Map) theMapObject;
// Verify the raw object that's been created
assertEquals(1l, theMap.get("one"));
assertEquals(2l, theMap.get("two"));
assertEquals(3l, theMap.get("three"));
// Verify that the provided object inspector can pull out these same values
StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
List<Object> z = oi.getStructFieldsDataAsList(row);
assertEquals(1, z.size());
StructField fieldRef = oi.getStructFieldRef("amap");
Map theMap2 = (Map) oi.getStructFieldData(row, fieldRef);
assertEquals(1l, theMap2.get("one"));
assertEquals(2l, theMap2.get("two"));
assertEquals(3l, theMap2.get("three"));
}
use of org.apache.hadoop.hive.serde2.SerDeException in project hive by apache.
the class TestAvroDeserializer method canDeserializeFixed.
// Fixed doesn't exist in Hive. Fixeds go in, lists of bytes go out.
@Test
public void canDeserializeFixed() throws SerDeException, IOException {
Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.FIXED_SCHEMA);
GenericData.Record record = new GenericData.Record(s);
byte[] bytes = "ANANCIENTBLUEBOX".getBytes();
record.put("hash", new GenericData.Fixed(s, bytes));
assertTrue(GENERIC_DATA.validate(s, record));
AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
AvroDeserializer de = new AvroDeserializer();
ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
assertEquals(1, row.size());
Object byteObject = row.get(0);
assertTrue(byteObject instanceof byte[]);
byte[] outBytes = (byte[]) byteObject;
// Verify the raw object that's been created
for (int i = 0; i < bytes.length; i++) {
assertEquals(bytes[i], outBytes[i]);
}
// Now go the correct way, through objectinspectors
StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
List<Object> fieldsDataAsList = oi.getStructFieldsDataAsList(row);
assertEquals(1, fieldsDataAsList.size());
StructField fieldRef = oi.getStructFieldRef("hash");
outBytes = (byte[]) oi.getStructFieldData(row, fieldRef);
for (int i = 0; i < outBytes.length; i++) {
assertEquals(bytes[i], outBytes[i]);
}
}
use of org.apache.hadoop.hive.serde2.SerDeException in project hive by apache.
the class TestLazyBinaryColumnarSerDe method testSerDeOuterNulls.
public void testSerDeOuterNulls() throws SerDeException {
StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
String cols = ObjectInspectorUtils.getFieldNames(oi);
Properties props = new Properties();
props.setProperty(serdeConstants.LIST_COLUMNS, cols);
props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
OuterStruct outerStruct = new OuterStruct();
BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
ObjectInspector out_oi = serde.getObjectInspector();
Object out_o = serde.deserialize(braw);
if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new SimpleMapEqualComparer())) {
System.out.println("expected = " + SerDeUtils.getJSONString(outerStruct, oi));
System.out.println("actual = " + SerDeUtils.getJSONString(out_o, out_oi));
fail("Deserialized object does not compare");
}
}
use of org.apache.hadoop.hive.serde2.SerDeException in project hive by apache.
the class TestLazyBinaryColumnarSerDe method testSerDeInnerNulls.
public void testSerDeInnerNulls() throws SerDeException {
StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
String cols = ObjectInspectorUtils.getFieldNames(oi);
Properties props = new Properties();
props.setProperty(serdeConstants.LIST_COLUMNS, cols);
props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
OuterStruct outerStruct = new OuterStruct();
outerStruct.mByte = 1;
outerStruct.mShort = 2;
outerStruct.mInt = 3;
outerStruct.mLong = 4l;
outerStruct.mFloat = 5.01f;
outerStruct.mDouble = 6.001d;
outerStruct.mString = "seven";
outerStruct.mBA = new byte[] { '3' };
InnerStruct is1 = new InnerStruct(null, 9l);
InnerStruct is2 = new InnerStruct(10, null);
outerStruct.mArray = new ArrayList<InnerStruct>(2);
outerStruct.mArray.add(is1);
outerStruct.mArray.add(is2);
outerStruct.mMap = new HashMap<String, InnerStruct>();
outerStruct.mMap.put(null, new InnerStruct(13, 14l));
outerStruct.mMap.put(new String("fifteen"), null);
outerStruct.mStruct = new InnerStruct(null, null);
BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
ObjectInspector out_oi = serde.getObjectInspector();
Object out_o = serde.deserialize(braw);
if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new SimpleMapEqualComparer())) {
System.out.println("expected = " + SerDeUtils.getJSONString(outerStruct, oi));
System.out.println("actual = " + SerDeUtils.getJSONString(out_o, out_oi));
fail("Deserialized object does not compare");
}
}
Aggregations