Examples with ObjectInspectorFactory.getReflectionObjectInspector org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector used on opensource projects

Example 46 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestLazyBinaryColumnarSerDe method testSerDeEmpties.

public void testSerDeEmpties() throws SerDeException {
    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
    String cols = ObjectInspectorUtils.getFieldNames(oi);
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, cols);
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
    OuterStruct outerStruct = new OuterStruct();
    outerStruct.mByte = 101;
    outerStruct.mShort = 2002;
    outerStruct.mInt = 3003;
    outerStruct.mLong = 4004l;
    outerStruct.mFloat = 5005.01f;
    outerStruct.mDouble = 6006.001d;
    outerStruct.mString = "";
    outerStruct.mBA = new byte[] { 'a' };
    outerStruct.mArray = new ArrayList<InnerStruct>();
    outerStruct.mMap = new TreeMap<String, InnerStruct>();
    outerStruct.mStruct = new InnerStruct(180018, 190019l);
    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
    ObjectInspector out_oi = serde.getObjectInspector();
    Object out_o = serde.deserialize(braw);
    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new SimpleMapEqualComparer())) {
        System.out.println("expected = " + SerDeUtils.getJSONString(outerStruct, oi));
        System.out.println("actual = " + SerDeUtils.getJSONString(out_o, out_oi));
        fail("Deserialized object does not compare");
    }
}

Example 47 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestLazyBinaryColumnarSerDe method testHandlingAlteredSchemas.

/**
   * HIVE-5788
   * <p>
   * Background: in cases of "add column", table metadata changes but data does not.  Columns
   * missing from the data but which are required by metadata are interpreted as null.
   * <p>
   * This tests the use-case of altering columns of a table with already some data, then adding more data
   * in the new schema, and seeing if this serde can to read both types of data from the resultant table.
   * @throws SerDeException
   */
public void testHandlingAlteredSchemas() throws SerDeException {
    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(BeforeStruct.class, ObjectInspectorOptions.JAVA);
    String cols = ObjectInspectorUtils.getFieldNames(oi);
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, cols);
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
    // serialize some data in the schema before it is altered.
    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
    BeforeStruct bs1 = new BeforeStruct();
    bs1.l1 = 1L;
    bs1.l2 = 2L;
    BytesRefArrayWritable braw1 = (BytesRefArrayWritable) serde.serialize(bs1, oi);
    // alter table add column: change the metadata
    oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(AfterStruct.class, ObjectInspectorOptions.JAVA);
    cols = ObjectInspectorUtils.getFieldNames(oi);
    props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, cols);
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
    serde = new LazyBinaryColumnarSerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
    // serialize some data in the schema after it is altered.
    AfterStruct as = new AfterStruct();
    as.l1 = 11L;
    as.l2 = 12L;
    as.l3 = 13L;
    BytesRefArrayWritable braw2 = (BytesRefArrayWritable) serde.serialize(as, oi);
    // fetch operator
    serde = new LazyBinaryColumnarSerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
    //fetch the row inserted before schema is altered and verify
    LazyBinaryColumnarStruct struct1 = (LazyBinaryColumnarStruct) serde.deserialize(braw1);
    oi = (StructObjectInspector) serde.getObjectInspector();
    List<Object> objs1 = oi.getStructFieldsDataAsList(struct1);
    Assert.assertEquals(((LongWritable) objs1.get(0)).get(), 1L);
    Assert.assertEquals(((LongWritable) objs1.get(1)).get(), 2L);
    Assert.assertNull(objs1.get(2));
    //fetch the row inserted after schema is altered and verify
    LazyBinaryColumnarStruct struct2 = (LazyBinaryColumnarStruct) serde.deserialize(braw2);
    List<Object> objs2 = struct2.getFieldsAsList();
    Assert.assertEquals(((LongWritable) objs2.get(0)).get(), 11L);
    Assert.assertEquals(((LongWritable) objs2.get(1)).get(), 12L);
    Assert.assertEquals(((LongWritable) objs2.get(2)).get(), 13L);
}

Example 48 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestLazyBinaryColumnarSerDe method testSerDe.

public void testSerDe() throws SerDeException {
    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
    String cols = ObjectInspectorUtils.getFieldNames(oi);
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, cols);
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
    OuterStruct outerStruct = new OuterStruct();
    outerStruct.mByte = 1;
    outerStruct.mShort = 2;
    outerStruct.mInt = 3;
    outerStruct.mLong = 4l;
    outerStruct.mFloat = 5.01f;
    outerStruct.mDouble = 6.001d;
    outerStruct.mString = "seven";
    outerStruct.mBA = new byte[] { '2' };
    InnerStruct is1 = new InnerStruct(8, 9l);
    InnerStruct is2 = new InnerStruct(10, 11l);
    outerStruct.mArray = new ArrayList<InnerStruct>(2);
    outerStruct.mArray.add(is1);
    outerStruct.mArray.add(is2);
    outerStruct.mMap = new TreeMap<String, InnerStruct>();
    outerStruct.mMap.put(new String("twelve"), new InnerStruct(13, 14l));
    outerStruct.mMap.put(new String("fifteen"), new InnerStruct(16, 17l));
    outerStruct.mStruct = new InnerStruct(18, 19l);
    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
    ObjectInspector out_oi = serde.getObjectInspector();
    Object out_o = serde.deserialize(braw);
    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new CrossMapEqualComparer())) {
        System.out.println("expected = " + SerDeUtils.getJSONString(outerStruct, oi));
        System.out.println("actual = " + SerDeUtils.getJSONString(out_o, out_oi));
        fail("Deserialized object does not compare");
    }
}

Example 49 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestStatsSerde method testLazyBinarySerDe.

/**
   * Test LazyBinarySerDe
   */
public void testLazyBinarySerDe() throws Throwable {
    try {
        System.out.println("test: testLazyBinarySerDe");
        int num = 1000;
        Random r = new Random(1234);
        MyTestClass[] rows = new MyTestClass[num];
        for (int i = 0; i < num; i++) {
            MyTestClass t = new MyTestClass();
            ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
            t.randomFill(r, extraTypeInfo);
            rows[i] = t;
        }
        StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
        Properties schema = new Properties();
        schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
        schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
        LazyBinarySerDe serDe = new LazyBinarySerDe();
        SerDeUtils.initializeSerDe(serDe, new Configuration(), schema, null);
        deserializeAndSerializeLazyBinary(serDe, rows, rowOI);
        System.out.println("test: testLazyBinarySerDe - OK");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 50 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestSimpleMapEqualComparer method testIncompatibleType.

public void testIncompatibleType() throws SerDeException, IOException {
    // empty maps
    StringTextMapHolder o1 = new StringTextMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    LazySerDeParameters serdeParams = new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
    SerDeUtils.initializeSerDe(serde, conf, tbl, null);
    ObjectInspector oi2 = serde.getObjectInspector();
    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertEquals(0, rc);
    // equal maps
    o1.mMap.put("42", new Text("The answer to Life, Universe And Everything"));
    o1.mMap.put("1729", new Text("A taxi cab number"));
    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertFalse(0 == rc);
}