use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class TestLazySimpleSerDe method testLazySimpleSerDeMissingColumns.
/**
* Test the LazySimpleSerDe class with missing columns.
*/
@Test
public void testLazySimpleSerDeMissingColumns() throws Throwable {
try {
// Create the SerDe
LazySimpleSerDe serDe = new LazySimpleSerDe();
Configuration conf = new Configuration();
Properties tbl = createProperties();
serDe.initialize(conf, tbl, null);
// Data
Text t = new Text("123\t456\t789\t1000\t5.3\t");
String s = "123\t456\t789\t1000\t5.3\t\tNULL\tNULL";
Object[] expectedFieldsData = { new ByteWritable((byte) 123), new ShortWritable((short) 456), new IntWritable(789), new LongWritable(1000), new DoubleWritable(5.3), new Text(""), null, null };
// Test
deserializeAndSerialize(serDe, t, s, expectedFieldsData);
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class TestGenericUDFRound method getExpectedResult.
@Override
public InspectableObject[] getExpectedResult() {
DataBuilder db = new DataBuilder();
db.setColumnNames("_col1", "_col2", "_col3", "_col4", "_col5", "_col6", "_col7", "_col8");
db.setColumnTypes(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableIntObjectInspector, PrimitiveObjectInspectorFactory.writableDoubleObjectInspector, PrimitiveObjectInspectorFactory.writableFloatObjectInspector, PrimitiveObjectInspectorFactory.writableByteObjectInspector, PrimitiveObjectInspectorFactory.writableShortObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector, PrimitiveObjectInspectorFactory.writableHiveDecimalObjectInspector);
db.addRow(null, new IntWritable(170), new DoubleWritable(1.1), new FloatWritable(32f), new ByteWritable((byte) 0), new ShortWritable((short) 1234), new LongWritable(123500L), new HiveDecimalWritable(HiveDecimal.create("983.724")));
db.addRow(new DoubleWritable(-200), null, null, new FloatWritable(0f), new ByteWritable((byte) 100), new ShortWritable((short) 551), new LongWritable(900L), new HiveDecimalWritable(HiveDecimal.create("983723.005")));
db.addRow(new DoubleWritable(500), new IntWritable(22345), new DoubleWritable(-23.000), new FloatWritable(-3f), new ByteWritable((byte) 100), new ShortWritable((short) 2321), new LongWritable(9200L), new HiveDecimalWritable(HiveDecimal.create("-932032.7")));
return db.createRows();
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class TestGenericUDFOPPositive method testByte.
@Test
public void testByte() throws HiveException {
GenericUDFOPPositive udf = new GenericUDFOPPositive();
ByteWritable input = new ByteWritable((byte) 4);
ObjectInspector[] inputOIs = { PrimitiveObjectInspectorFactory.writableByteObjectInspector };
DeferredObject[] args = { new DeferredJavaObject(input) };
PrimitiveObjectInspector oi = (PrimitiveObjectInspector) udf.initialize(inputOIs);
Assert.assertEquals(TypeInfoFactory.byteTypeInfo, oi.getTypeInfo());
ByteWritable res = (ByteWritable) udf.evaluate(args);
Assert.assertEquals((byte) 4, res.get());
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class TestLazyArrayMapStruct method testLazyArray.
/**
* Test the LazyArray class.
*/
@Test
public void testLazyArray() throws Throwable {
try {
// Array of Byte
Text nullSequence = new Text("\\N");
ObjectInspector oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils.getTypeInfosFromTypeString("array<tinyint>").get(0), new byte[] { (byte) 1 }, 0, nullSequence, false, (byte) 0);
LazyArray b = (LazyArray) LazyFactory.createLazyObject(oi);
byte[] data = new byte[] { '-', '1', 1, '\\', 'N', 1, '8' };
TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
assertNull(b.getListElementObject(-1));
assertEquals(new ByteWritable((byte) -1), ((LazyByte) b.getListElementObject(0)).getWritableObject());
assertEquals(new ByteWritable((byte) -1), ((LazyByte) b.getList().get(0)).getWritableObject());
assertNull(b.getListElementObject(1));
assertNull(b.getList().get(1));
assertEquals(new ByteWritable((byte) 8), ((LazyByte) b.getListElementObject(2)).getWritableObject());
assertEquals(new ByteWritable((byte) 8), ((LazyByte) b.getList().get(2)).getWritableObject());
assertNull(b.getListElementObject(3));
assertEquals(3, b.getList().size());
// Array of String
oi = LazyFactory.createLazyObjectInspector(TypeInfoUtils.getTypeInfosFromTypeString("array<string>").get(0), new byte[] { (byte) '\t' }, 0, nullSequence, false, (byte) 0);
b = (LazyArray) LazyFactory.createLazyObject(oi);
data = new byte[] { 'a', 'b', '\t', 'c', '\t', '\\', 'N', '\t', '\t', 'd' };
// Note: the first and last element of the byte[] are NOT used
TestLazyPrimitive.initLazyObject(b, data, 1, data.length - 2);
assertNull(b.getListElementObject(-1));
assertEquals(new Text("b"), ((LazyString) b.getListElementObject(0)).getWritableObject());
assertEquals(new Text("b"), ((LazyString) b.getList().get(0)).getWritableObject());
assertEquals(new Text("c"), ((LazyString) b.getListElementObject(1)).getWritableObject());
assertEquals(new Text("c"), ((LazyString) b.getList().get(1)).getWritableObject());
assertNull((b.getListElementObject(2)));
assertNull((b.getList().get(2)));
assertEquals(new Text(""), ((LazyString) b.getListElementObject(3)).getWritableObject());
assertEquals(new Text(""), ((LazyString) b.getList().get(3)).getWritableObject());
assertEquals(new Text(""), ((LazyString) b.getListElementObject(4)).getWritableObject());
assertEquals(new Text(""), ((LazyString) b.getList().get(4)).getWritableObject());
assertNull((b.getListElementObject(5)));
assertEquals(5, b.getList().size());
// -- HIVE-4149
b = (LazyArray) LazyFactory.createLazyObject(oi);
data = new byte[] { 'a', '\t', '\\', 'N' };
TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
assertEquals(new Text("a"), ((LazyString) b.getListElementObject(0)).getWritableObject());
assertNull(b.getListElementObject(1));
data = new byte[] { '\\', 'N', '\t', 'a' };
TestLazyPrimitive.initLazyObject(b, data, 0, data.length);
assertNull(b.getListElementObject(0));
// twice (returns not cleaned cache)
assertNull(b.getListElementObject(0));
assertEquals(new Text("a"), ((LazyString) b.getListElementObject(1)).getWritableObject());
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.hive.serde2.io.ByteWritable in project hive by apache.
the class TestLazyArrayMapStruct method testNestedinArrayAtLevel.
/**
* @param nestingLevel
* @param dtype
* @param tableProp
* @throws SerDeException
*/
private void testNestedinArrayAtLevel(int nestingLevel, ObjectInspector.Category dtype, Properties tableProp) throws SerDeException {
// create type with nestingLevel levels of nesting
// set inner schema for dtype
String inSchema = null;
switch(dtype) {
case LIST:
inSchema = "array<tinyint>";
break;
case MAP:
inSchema = "map<string,int>";
break;
case STRUCT:
inSchema = "struct<s:string,i:tinyint>";
break;
case UNION:
inSchema = "uniontype<string,tinyint>";
break;
default:
fail("type not supported by test case");
}
StringBuilder schema = new StringBuilder(inSchema);
for (int i = 0; i < nestingLevel - 1; i++) {
schema.insert(0, "array<");
schema.append(">");
}
System.err.println("Testing nesting level " + nestingLevel + ". Using schema " + schema);
// Create the SerDe
LazySimpleSerDe serDe = new LazySimpleSerDe();
Configuration conf = new Configuration();
tableProp.setProperty("columns", "narray");
tableProp.setProperty("columns.types", schema.toString());
serDe.initialize(conf, tableProp, null);
LazySerDeParameters serdeParams = new LazySerDeParameters(conf, tableProp, LazySimpleSerDe.class.getName());
// create the serialized string for type
byte[] separators = serdeParams.getSeparators();
System.err.println("Using separator " + (char) separators[nestingLevel]);
byte[] serializedRow = null;
switch(dtype) {
case LIST:
serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
break;
case MAP:
byte kvSep = separators[nestingLevel + 1];
byte kvPairSep = separators[nestingLevel];
serializedRow = new byte[] { '1', kvSep, '1', kvPairSep, '2', kvSep, '2' };
break;
case STRUCT:
serializedRow = new byte[] { '8', separators[nestingLevel], '9' };
break;
case UNION:
serializedRow = new byte[] { '0', separators[nestingLevel], '9' };
break;
default:
fail("type not supported by test case");
}
// create LazyStruct with serialized string with expected separators
StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
LazyStruct struct = (LazyStruct) LazyFactory.createLazyObject(oi);
TestLazyPrimitive.initLazyObject(struct, serializedRow, 0, serializedRow.length);
// Get fields out of the lazy struct and check if they match expected
// results
// Get first level array
LazyArray array = (LazyArray) struct.getField(0);
// Peel off the n-1 levels to get to the underlying array
for (int i = 0; i < nestingLevel - 2; i++) {
array = (LazyArray) array.getListElementObject(0);
}
// verify the serialized format for dtype
switch(dtype) {
case LIST:
LazyArray array1 = (LazyArray) array.getListElementObject(0);
// check elements of the innermost array
assertEquals(2, array1.getListLength());
assertEquals(new ByteWritable((byte) 8), ((LazyByte) array1.getListElementObject(0)).getWritableObject());
assertEquals(new ByteWritable((byte) 9), ((LazyByte) array1.getListElementObject(1)).getWritableObject());
break;
case MAP:
LazyMap lazyMap = (LazyMap) array.getListElementObject(0);
Map map = lazyMap.getMap();
System.err.println(map);
assertEquals(2, map.size());
Iterator<Map.Entry<LazyString, LazyInteger>> it = map.entrySet().iterator();
Entry<LazyString, LazyInteger> e1 = it.next();
assertEquals(e1.getKey().getWritableObject(), new Text(new byte[] { '1' }));
assertEquals(e1.getValue().getWritableObject(), new IntWritable(1));
Entry<LazyString, LazyInteger> e2 = it.next();
assertEquals(e2.getKey().getWritableObject(), new Text(new byte[] { '2' }));
assertEquals(e2.getValue().getWritableObject(), new IntWritable(2));
break;
case STRUCT:
LazyStruct innerStruct = (LazyStruct) array.getListElementObject(0);
// check elements of the innermost struct
assertEquals(2, innerStruct.getFieldsAsList().size());
assertEquals(new Text(new byte[] { '8' }), ((LazyString) innerStruct.getField(0)).getWritableObject());
assertEquals(new ByteWritable((byte) 9), ((LazyByte) innerStruct.getField(1)).getWritableObject());
break;
case UNION:
LazyUnion lazyUnion = (LazyUnion) array.getListElementObject(0);
// check elements of the innermost union
assertEquals(new Text(new byte[] { '9' }), ((LazyString) lazyUnion.getField()).getWritableObject());
break;
default:
fail("type not supported by test case");
}
// test serialization
Text serializedText = (Text) serDe.serialize(struct.getObject(), serDe.getObjectInspector());
org.junit.Assert.assertArrayEquals(serializedRow, serializedText.getBytes());
}
Aggregations