use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableLongObjectInspector in project hive by apache.
the class VectorUDAFAvgDecimal method initPartialResultInspector.
private void initPartialResultInspector() {
// the output type of the vectorized partial aggregate must match the
// expected type for the row-mode aggregation
// For decimal, the type is "same number of integer digits and 4 more decimal digits"
DecimalTypeInfo dtiSum = GenericUDAFAverage.deriveSumFieldTypeInfo(inputPrecision, inputScale);
this.sumScale = (short) dtiSum.scale();
this.sumPrecision = (short) dtiSum.precision();
List<ObjectInspector> foi = new ArrayList<ObjectInspector>();
foi.add(PrimitiveObjectInspectorFactory.writableLongObjectInspector);
foi.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(dtiSum));
List<String> fname = new ArrayList<String>();
fname.add("count");
fname.add("sum");
soi = ObjectInspectorFactory.getStandardStructObjectInspector(fname, foi);
}
use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableLongObjectInspector in project hive by apache.
the class TestObjectInspectorConverters method testObjectInspectorConverters.
public void testObjectInspectorConverters() throws Throwable {
try {
// Boolean
Converter booleanConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
assertEquals("BooleanConverter", new BooleanWritable(false), booleanConverter.convert(Integer.valueOf(0)));
assertEquals("BooleanConverter", new BooleanWritable(true), booleanConverter.convert(Integer.valueOf(1)));
assertEquals("BooleanConverter", null, booleanConverter.convert(null));
// Byte
Converter byteConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableByteObjectInspector);
assertEquals("ByteConverter", new ByteWritable((byte) 0), byteConverter.convert(Integer.valueOf(0)));
assertEquals("ByteConverter", new ByteWritable((byte) 1), byteConverter.convert(Integer.valueOf(1)));
assertEquals("ByteConverter", null, byteConverter.convert(null));
// Short
Converter shortConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableShortObjectInspector);
assertEquals("ShortConverter", new ShortWritable((short) 0), shortConverter.convert(Integer.valueOf(0)));
assertEquals("ShortConverter", new ShortWritable((short) 1), shortConverter.convert(Integer.valueOf(1)));
assertEquals("ShortConverter", null, shortConverter.convert(null));
// Int
Converter intConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableIntObjectInspector);
assertEquals("IntConverter", new IntWritable(0), intConverter.convert(Integer.valueOf(0)));
assertEquals("IntConverter", new IntWritable(1), intConverter.convert(Integer.valueOf(1)));
assertEquals("IntConverter", null, intConverter.convert(null));
// Long
Converter longConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector);
assertEquals("LongConverter", new LongWritable(0), longConverter.convert(Integer.valueOf(0)));
assertEquals("LongConverter", new LongWritable(1), longConverter.convert(Integer.valueOf(1)));
assertEquals("LongConverter", null, longConverter.convert(null));
// Float
Converter floatConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableFloatObjectInspector);
assertEquals("LongConverter", new FloatWritable(0), floatConverter.convert(Integer.valueOf(0)));
assertEquals("LongConverter", new FloatWritable(1), floatConverter.convert(Integer.valueOf(1)));
assertEquals("LongConverter", null, floatConverter.convert(null));
// Double
Converter doubleConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableDoubleObjectInspector);
assertEquals("DoubleConverter", new DoubleWritable(0), doubleConverter.convert(Integer.valueOf(0)));
assertEquals("DoubleConverter", new DoubleWritable(1), doubleConverter.convert(Integer.valueOf(1)));
assertEquals("DoubleConverter", null, doubleConverter.convert(null));
// Char
Converter charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
assertEquals("CharConverter", new HiveChar("TRUE", -1), charConverter.convert(Boolean.valueOf(true)));
assertEquals("CharConverter", new HiveChar("FALSE", -1), charConverter.convert(Boolean.valueOf(false)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("TRUE", -1)), charConverter.convert(Boolean.valueOf(true)));
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("FALSE", -1)), charConverter.convert(Boolean.valueOf(false)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
assertEquals("CharConverter", new HiveChar("0", -1), charConverter.convert(Integer.valueOf(0)));
assertEquals("CharConverter", new HiveChar("1", -1), charConverter.convert(Integer.valueOf(1)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("0", -1)), charConverter.convert(Integer.valueOf(0)));
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("1", -1)), charConverter.convert(Integer.valueOf(1)));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaHiveCharObjectInspector);
assertEquals("CharConverter", new HiveChar("hive", -1), charConverter.convert(String.valueOf("hive")));
charConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableHiveCharObjectInspector);
assertEquals("CharConverter", new HiveCharWritable(new HiveChar("hive", -1)), charConverter.convert(String.valueOf("hive")));
// VarChar
Converter varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarchar("TRUE", -1), varcharConverter.convert(Boolean.valueOf(true)));
assertEquals("VarCharConverter", new HiveVarchar("FALSE", -1), varcharConverter.convert(Boolean.valueOf(false)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("TRUE", -1)), varcharConverter.convert(Boolean.valueOf(true)));
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("FALSE", -1)), varcharConverter.convert(Boolean.valueOf(false)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarchar("0", -1), varcharConverter.convert(Integer.valueOf(0)));
assertEquals("VarCharConverter", new HiveVarchar("1", -1), varcharConverter.convert(Integer.valueOf(1)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("0", -1)), varcharConverter.convert(Integer.valueOf(0)));
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("1", -1)), varcharConverter.convert(Integer.valueOf(1)));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarchar("hive", -1), varcharConverter.convert(String.valueOf("hive")));
varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("hive", -1)), varcharConverter.convert(String.valueOf("hive")));
// Text
Converter textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("0"), textConverter.convert(Integer.valueOf(0)));
assertEquals("TextConverter", new Text("1"), textConverter.convert(Integer.valueOf(1)));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.writableBinaryObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("hive"), textConverter.convert(new BytesWritable(new byte[] { (byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e' })));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.writableStringObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("hive"), textConverter.convert(new Text("hive")));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("hive"), textConverter.convert(new String("hive")));
assertEquals("TextConverter", null, textConverter.convert(null));
textConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector);
assertEquals("TextConverter", new Text("100.001"), textConverter.convert(HiveDecimal.create("100.001")));
assertEquals("TextConverter", null, textConverter.convert(null));
// Binary
Converter baConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
assertEquals("BAConverter", new BytesWritable(new byte[] { (byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e' }), baConverter.convert("hive"));
assertEquals("BAConverter", null, baConverter.convert(null));
baConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.writableStringObjectInspector, PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
assertEquals("BAConverter", new BytesWritable(new byte[] { (byte) 'h', (byte) 'i', (byte) 'v', (byte) 'e' }), baConverter.convert(new Text("hive")));
assertEquals("BAConverter", null, baConverter.convert(null));
// Union
ArrayList<String> fieldNames = new ArrayList<String>();
fieldNames.add("firstInteger");
fieldNames.add("secondString");
fieldNames.add("thirdBoolean");
ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
ArrayList<String> fieldNames2 = new ArrayList<String>();
fieldNames2.add("firstString");
fieldNames2.add("secondInteger");
fieldNames2.add("thirdBoolean");
ArrayList<ObjectInspector> fieldObjectInspectors2 = new ArrayList<ObjectInspector>();
fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
fieldObjectInspectors2.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
Converter unionConverter0 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
Object convertedObject0 = unionConverter0.convert(new StandardUnion((byte) 0, 1));
StandardUnion expectedObject0 = new StandardUnion();
expectedObject0.setTag((byte) 0);
expectedObject0.setObject("1");
assertEquals(expectedObject0, convertedObject0);
Converter unionConverter1 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
Object convertedObject1 = unionConverter1.convert(new StandardUnion((byte) 1, "1"));
StandardUnion expectedObject1 = new StandardUnion();
expectedObject1.setTag((byte) 1);
expectedObject1.setObject(1);
assertEquals(expectedObject1, convertedObject1);
Converter unionConverter2 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
Object convertedObject2 = unionConverter2.convert(new StandardUnion((byte) 2, true));
StandardUnion expectedObject2 = new StandardUnion();
expectedObject2.setTag((byte) 2);
expectedObject2.setObject(true);
assertEquals(expectedObject2, convertedObject2);
// Union (extra fields)
ArrayList<String> fieldNamesExtra = new ArrayList<String>();
fieldNamesExtra.add("firstInteger");
fieldNamesExtra.add("secondString");
fieldNamesExtra.add("thirdBoolean");
ArrayList<ObjectInspector> fieldObjectInspectorsExtra = new ArrayList<ObjectInspector>();
fieldObjectInspectorsExtra.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
fieldObjectInspectorsExtra.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectorsExtra.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
ArrayList<String> fieldNamesExtra2 = new ArrayList<String>();
fieldNamesExtra2.add("firstString");
fieldNamesExtra2.add("secondInteger");
ArrayList<ObjectInspector> fieldObjectInspectorsExtra2 = new ArrayList<ObjectInspector>();
fieldObjectInspectorsExtra2.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
fieldObjectInspectorsExtra2.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
Converter unionConverterExtra = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra), ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra2));
Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnion((byte) 2, true));
StandardUnion expectedObjectExtra = new StandardUnion();
expectedObjectExtra.setTag((byte) -1);
expectedObjectExtra.setObject(null);
// we should get back null
assertEquals(expectedObjectExtra, convertedObjectExtra);
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableLongObjectInspector in project flink by apache.
the class GenericUDFLegacyGroupingID method initialize.
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
// we accept two arguments: the new GROUPING__ID and the number of GBY expressions
if (arguments.length != 2) {
throw new UDFArgumentLengthException("Expect 2 arguments but actually got " + arguments.length);
}
if (arguments[0].getCategory() != ObjectInspector.Category.PRIMITIVE) {
throw new UDFArgumentTypeException(0, "First argument should be primitive type");
}
if (arguments[1].getCategory() != ObjectInspector.Category.PRIMITIVE) {
throw new UDFArgumentTypeException(1, "Second argument should be primitive type");
}
groupingIdOI = (PrimitiveObjectInspector) arguments[0];
if (groupingIdOI.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.LONG) {
throw new UDFArgumentTypeException(0, "First argument should be a long");
}
PrimitiveObjectInspector numExprOI = (PrimitiveObjectInspector) arguments[1];
if (numExprOI.getPrimitiveCategory() != PrimitiveObjectInspector.PrimitiveCategory.INT) {
throw new UDFArgumentTypeException(1, "Second argument should be an int");
}
if (!(numExprOI instanceof ConstantObjectInspector)) {
throw new UDFArgumentTypeException(1, "Second argument should be a constant");
}
numExprs = PrimitiveObjectInspectorUtils.getInt(((ConstantObjectInspector) numExprOI).getWritableConstantValue(), numExprOI);
if (numExprs < 1 || numExprs > 64) {
throw new UDFArgumentException("Number of GROUP BY expressions out of range: " + numExprs);
}
return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
}
use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableLongObjectInspector in project hive by apache.
the class TestDeserializer method testMapDeserialize.
@Test
public void testMapDeserialize() {
Schema schema = new Schema(optional(1, "map_type", Types.MapType.ofOptional(2, 3, Types.LongType.get(), Types.StringType.get())));
StructObjectInspector inspector = ObjectInspectorFactory.getStandardStructObjectInspector(Arrays.asList("map_type"), Arrays.asList(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.writableLongObjectInspector, PrimitiveObjectInspectorFactory.writableStringObjectInspector)));
Deserializer deserializer = new Deserializer.Builder().schema(schema).writerInspector((StructObjectInspector) IcebergObjectInspector.create(schema)).sourceInspector(inspector).build();
Record expected = GenericRecord.create(schema);
expected.set(0, Collections.singletonMap(1L, "Taylor"));
MapWritable map = new MapWritable();
map.put(new LongWritable(1L), new Text("Taylor"));
Object[] data = new Object[] { map };
Record actual = deserializer.deserialize(data);
Assert.assertEquals(expected, actual);
}
use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.writableLongObjectInspector in project hive by apache.
the class TestMurmurHashExpression method testMurmurHashIntColIntCol.
@Test
public void testMurmurHashIntColIntCol() throws HiveException {
LongColumnVector cvInt1 = (LongColumnVector) ColumnVectorGenUtil.generateColumnVector(TypeInfoFactory.getPrimitiveTypeInfo("int"), false, false, SIZE, rand);
LongColumnVector cvInt2 = (LongColumnVector) ColumnVectorGenUtil.generateColumnVector(TypeInfoFactory.getPrimitiveTypeInfo("int"), false, false, SIZE, rand);
VectorizedRowBatch vrb = new VectorizedRowBatch(3, SIZE);
vrb.cols[0] = cvInt1;
vrb.cols[1] = cvInt2;
vrb.cols[2] = new LongColumnVector(SIZE);
new MurmurHashIntColIntCol(0, 1, 2).evaluate(vrb);
// non-repeating
Assert.assertEquals(false, vrb.cols[2].isRepeating);
for (int i = 0; i < SIZE; i++) {
Assert.assertEquals(ObjectInspectorUtils.getBucketHashCode(new Object[] { new LongWritable(cvInt1.vector[i]), new LongWritable(cvInt2.vector[i]) }, new ObjectInspector[] { PrimitiveObjectInspectorFactory.writableLongObjectInspector, PrimitiveObjectInspectorFactory.writableLongObjectInspector }), ((LongColumnVector) vrb.cols[2]).vector[i]);
}
}
Aggregations