Examples with ExtraTypeInfo org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo used on opensource projects

Example 6 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestStatsSerde method testLazyBinarySerDe.

/**
   * Test LazyBinarySerDe
   */
public void testLazyBinarySerDe() throws Throwable {
    try {
        System.out.println("test: testLazyBinarySerDe");
        int num = 1000;
        Random r = new Random(1234);
        MyTestClass[] rows = new MyTestClass[num];
        for (int i = 0; i < num; i++) {
            MyTestClass t = new MyTestClass();
            ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
            t.randomFill(r, extraTypeInfo);
            rows[i] = t;
        }
        StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
        Properties schema = new Properties();
        schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
        schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
        LazyBinarySerDe serDe = new LazyBinarySerDe();
        SerDeUtils.initializeSerDe(serDe, new Configuration(), schema, null);
        deserializeAndSerializeLazyBinary(serDe, rows, rowOI);
        System.out.println("test: testLazyBinarySerDe - OK");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 7 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestLazyBinarySerDe method testShorterSchemaDeserialization.

/**
   * Test shorter schema deserialization where a bigger struct is serialized and
   * it is then deserialized with a smaller struct. Here the serialized struct
   * has 10 fields and we deserialized to a struct of 9 fields.
   */
private void testShorterSchemaDeserialization(Random r) throws Throwable {
    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClassBigger.class, ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();
    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();
    int num = 100;
    for (int itest = 0; itest < num; itest++) {
        MyTestClassBigger t = new MyTestClassBigger();
        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
        t.randomFill(r, extraTypeInfo);
        BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
        Object output = serde2.deserialize(bw);
        if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
            System.out.println("structs      = " + SerDeUtils.getJSONString(t, rowOI1));
            System.out.println("deserialized = " + SerDeUtils.getJSONString(output, serdeOI2));
            System.out.println("serialized   = " + TestBinarySortableSerDe.hexString(bw));
            assertEquals(t, output);
        }
    }
}

Example 8 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestLazyBinarySerDe method testLongerSchemaDeserialization.

/**
   * Test longer schema deserialization where a smaller struct is serialized and
   * it is then deserialized with a bigger struct Here the serialized struct has
   * 9 fields and we deserialized to a struct of 10 fields.
   */
void testLongerSchemaDeserialization(Random r) throws Throwable {
    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();
    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClassBigger.class, ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();
    int num = 100;
    for (int itest = 0; itest < num; itest++) {
        MyTestClass t = new MyTestClass();
        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
        t.randomFill(r, extraTypeInfo);
        BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
        Object output = serde2.deserialize(bw);
        if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
            System.out.println("structs      = " + SerDeUtils.getJSONString(t, rowOI1));
            System.out.println("deserialized = " + SerDeUtils.getJSONString(output, serdeOI2));
            System.out.println("serialized   = " + TestBinarySortableSerDe.hexString(bw));
            assertEquals(t, output);
        }
    }
}

Example 9 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestLazyBinarySerDe method testLazyBinarySerDe.

/**
   * The test entrance function.
   *
   * @throws Throwable
   */
public void testLazyBinarySerDe() throws Throwable {
    try {
        System.out.println("Beginning Test TestLazyBinarySerDe:");
        // generate the data
        int num = 1000;
        Random r = new Random(1234);
        MyTestClass[] rows = new MyTestClass[num];
        for (int i = 0; i < num; i++) {
            MyTestClass t = new MyTestClass();
            ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
            t.randomFill(r, extraTypeInfo);
            rows[i] = t;
        }
        StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
        // call the tests
        // 1/ test LazyBinarySerDe
        testLazyBinarySerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes));
        // 2/ test LazyBinaryMap
        testLazyBinaryMap(r);
        // 3/ test serialization and deserialization with different schemas
        testShorterSchemaDeserialization(r);
        // 4/ test serialization and deserialization with different schemas
        testLongerSchemaDeserialization(r);
        // 5/ test serialization and deserialization with different schemas
        testShorterSchemaDeserialization1(r);
        // 6/ test serialization and deserialization with different schemas
        testLongerSchemaDeserialization1(r);
        System.out.println("Test TestLazyBinarySerDe passed!");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}