use of org.apache.hadoop.hive.serde2.lazy.LazyUnion in project hive by apache.
the class TestLazyArrayMapStruct method testNestedinArrayAtLevel.
/**
* @param nestingLevel
* @param dtype
* @param tableProp
* @throws SerDeException
*/
private void testNestedinArrayAtLevel(int nestingLevel, ObjectInspector.Category dtype, Properties tableProp) throws SerDeException {
//create type with nestingLevel levels of nesting
//set inner schema for dtype
String inSchema = null;
switch(dtype) {
case LIST:
inSchema = "array<tinyint>";
break;
case MAP:
inSchema = "map<string,int>";
break;
case STRUCT:
inSchema = "struct<s:string,i:tinyint>";
break;
case UNION:
inSchema = "uniontype<string,tinyint>";
break;
default:
fail("type not supported by test case");
}
StringBuilder schema = new StringBuilder(inSchema);
for (int i = 0; i < nestingLevel - 1; i++) {
schema.insert(0, "array<");
schema.append(">");
}
System.err.println("Testing nesting level " + nestingLevel + ". Using schema " + schema);
// Create the SerDe
LazySimpleSerDe serDe = new LazySimpleSerDe();
Configuration conf = new Configuration();
tableProp.setProperty("columns", "narray");
tableProp.setProperty("columns.types", schema.toString());
SerDeUtils.initializeSerDe(serDe, conf, tableProp, null);
LazySerDeParameters serdeParams = new LazySerDeParameters(conf, tableProp, LazySimpleSerDe.class.getName());
//create the serialized string for type
byte[] separators = serdeParams.getSeparators();
System.err.println("Using separator " + (char) separators[nestingLevel]);
byte[] serializedRow = null;
switch(dtype) {
case LIST:
serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
break;
case MAP:
byte kvSep = separators[nestingLevel + 1];
byte kvPairSep = separators[nestingLevel];
serializedRow = new byte[] { '1', kvSep, '1', kvPairSep, '2', kvSep, '2' };
break;
case STRUCT:
serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
break;
case UNION:
serializedRow = new byte[] { '0', separators[nestingLevel], '9' };
break;
default:
fail("type not supported by test case");
}
//create LazyStruct with serialized string with expected separators
StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(oi);
TestLazyPrimitive.initLazyObject(struct, serializedRow, 0, serializedRow.length);
//Get fields out of the lazy struct and check if they match expected
// results
//Get first level array
LazyArray array = (LazyArray) struct.getField(0);
//Peel off the n-1 levels to get to the underlying array
for (int i = 0; i < nestingLevel - 2; i++) {
array = (LazyArray) array.getListElementObject(0);
}
//verify the serialized format for dtype
switch(dtype) {
case LIST:
LazyArray array1 = (LazyArray) array.getListElementObject(0);
//check elements of the innermost array
assertEquals(2, array1.getListLength());
assertEquals(new ByteWritable((byte) 8), ((LazyByte) array1.getListElementObject(0)).getWritableObject());
assertEquals(new ByteWritable((byte) 9), ((LazyByte) array1.getListElementObject(1)).getWritableObject());
break;
case MAP:
LazyMap lazyMap = (LazyMap) array.getListElementObject(0);
Map map = lazyMap.getMap();
System.err.println(map);
assertEquals(2, map.size());
Iterator<Map.Entry<LazyString, LazyInteger>> it = map.entrySet().iterator();
Entry<LazyString, LazyInteger> e1 = it.next();
assertEquals(e1.getKey().getWritableObject(), new Text(new byte[] { '1' }));
assertEquals(e1.getValue().getWritableObject(), new IntWritable(1));
Entry<LazyString, LazyInteger> e2 = it.next();
assertEquals(e2.getKey().getWritableObject(), new Text(new byte[] { '2' }));
assertEquals(e2.getValue().getWritableObject(), new IntWritable(2));
break;
case STRUCT:
LazyStruct innerStruct = (LazyStruct) array.getListElementObject(0);
//check elements of the innermost struct
assertEquals(2, innerStruct.getFieldsAsList().size());
assertEquals(new Text(new byte[] { '8' }), ((LazyString) innerStruct.getField(0)).getWritableObject());
assertEquals(new ByteWritable((byte) 9), ((LazyByte) innerStruct.getField(1)).getWritableObject());
break;
case UNION:
LazyUnion lazyUnion = (LazyUnion) array.getListElementObject(0);
//check elements of the innermost union
assertEquals(new Text(new byte[] { '9' }), ((LazyString) lazyUnion.getField()).getWritableObject());
break;
default:
fail("type not supported by test case");
}
//test serialization
Text serializedText = (Text) serDe.serialize(struct.getObject(), serDe.getObjectInspector());
org.junit.Assert.assertArrayEquals(serializedRow, serializedText.getBytes());
}
use of org.apache.hadoop.hive.serde2.lazy.LazyUnion in project hive by apache.
the class AvroLazyObjectInspector method toLazyUnionObject.
/**
* Convert the given object to a lazy object using the given {@link ObjectInspector}
*
* @param obj Object to be converted to a {@link LazyObject}
* @param oi ObjectInspector used for the conversion
* @return the created {@link LazyObject lazy object}
* */
private Object toLazyUnionObject(Object obj, ObjectInspector objectInspector) {
if (obj == null) {
return null;
}
if (!(objectInspector instanceof LazyUnionObjectInspector)) {
throw new IllegalArgumentException("Invalid objectinspector found. Expected LazyUnionObjectInspector, Found " + objectInspector.getClass());
}
StandardUnion standardUnion = (StandardUnion) obj;
LazyUnionObjectInspector lazyUnionOI = (LazyUnionObjectInspector) objectInspector;
// Grab the tag and the field
byte tag = standardUnion.getTag();
Object field = standardUnion.getObject();
ObjectInspector fieldOI = lazyUnionOI.getObjectInspectors().get(tag);
// convert to lazy object
Object convertedObj = null;
if (field != null) {
convertedObj = toLazyObject(field, fieldOI);
}
if (convertedObj == null) {
return null;
}
return new LazyUnion(lazyUnionOI, tag, convertedObj);
}
Aggregations