Examples with ExtraTypeInfo org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo used on opensource projects

Example 1 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestLazyBinarySerDe method testLongerSchemaDeserialization1.

/**
   * Test longer schema deserialization where a smaller struct is serialized and
   * it is then deserialized with a bigger struct Here the serialized struct has
   * 8 fields and we deserialized to a struct of 9 fields.
   */
void testLongerSchemaDeserialization1(Random r) throws Throwable {
    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClassSmaller.class, ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();
    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();
    int num = 100;
    for (int itest = 0; itest < num; itest++) {
        MyTestClassSmaller t = new MyTestClassSmaller();
        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
        t.randomFill(r, extraTypeInfo);
        BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
        Object output = serde2.deserialize(bw);
        if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
            System.out.println("structs      = " + SerDeUtils.getJSONString(t, rowOI1));
            System.out.println("deserialized = " + SerDeUtils.getJSONString(output, serdeOI2));
            System.out.println("serialized   = " + TestBinarySortableSerDe.hexString(bw));
            assertEquals(t, output);
        }
    }
}

Example 2 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestLazyBinarySerDe method testShorterSchemaDeserialization1.

/**
   * Test shorter schema deserialization where a bigger struct is serialized and
   * it is then deserialized with a smaller struct. Here the serialized struct
   * has 9 fields and we deserialized to a struct of 8 fields.
   */
private void testShorterSchemaDeserialization1(Random r) throws Throwable {
    StructObjectInspector rowOI1 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
    String fieldNames1 = ObjectInspectorUtils.getFieldNames(rowOI1);
    String fieldTypes1 = ObjectInspectorUtils.getFieldTypes(rowOI1);
    AbstractSerDe serde1 = getSerDe(fieldNames1, fieldTypes1);
    serde1.getObjectInspector();
    StructObjectInspector rowOI2 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClassSmaller.class, ObjectInspectorOptions.JAVA);
    String fieldNames2 = ObjectInspectorUtils.getFieldNames(rowOI2);
    String fieldTypes2 = ObjectInspectorUtils.getFieldTypes(rowOI2);
    AbstractSerDe serde2 = getSerDe(fieldNames2, fieldTypes2);
    ObjectInspector serdeOI2 = serde2.getObjectInspector();
    int num = 100;
    for (int itest = 0; itest < num; itest++) {
        MyTestClass t = new MyTestClass();
        ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
        t.randomFill(r, extraTypeInfo);
        BytesWritable bw = (BytesWritable) serde1.serialize(t, rowOI1);
        Object output = serde2.deserialize(bw);
        if (0 != compareDiffSizedStructs(t, rowOI1, output, serdeOI2)) {
            System.out.println("structs      = " + SerDeUtils.getJSONString(t, rowOI1));
            System.out.println("deserialized = " + SerDeUtils.getJSONString(output, serdeOI2));
            System.out.println("serialized   = " + TestBinarySortableSerDe.hexString(bw));
            assertEquals(t, output);
        }
    }
}

Example 3 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestLazySimpleSerDe method testSerDeParameters.

/**
   * Tests the deprecated usage of SerDeParameters.
   *
   */
@Test
@SuppressWarnings("deprecation")
public void testSerDeParameters() throws SerDeException, IOException {
    // Setup
    LazySimpleSerDe serDe = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    MyTestClass row = new MyTestClass();
    ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
    row.randomFill(new Random(1234), extraTypeInfo);
    StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
    String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
    String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
    Properties schema = new Properties();
    schema.setProperty(serdeConstants.LIST_COLUMNS, fieldNames);
    schema.setProperty(serdeConstants.LIST_COLUMN_TYPES, fieldTypes);
    SerDeUtils.initializeSerDe(serDe, conf, schema, null);
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, schema, "testSerdeName");
    // Test
    LazyStruct data = (LazyStruct) serializeAndDeserialize(row, rowOI, serDe, serdeParams);
    assertEquals((boolean) row.myBool, ((LazyBoolean) data.getField(0)).getWritableObject().get());
    assertEquals((int) row.myInt, ((LazyInteger) data.getField(3)).getWritableObject().get());
}

Example 4 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class MyTestPrimitiveClass method getPrimitiveTypeInfo.

public static PrimitiveTypeInfo getPrimitiveTypeInfo(int index, ExtraTypeInfo extraTypeInfo) {
    PrimitiveCategory primitiveCategory = getPrimitiveCategory(index);
    String typeName;
    switch(primitiveCategory) {
        case BYTE:
            typeName = "tinyint";
            break;
        case SHORT:
            typeName = "smallint";
            break;
        case LONG:
            typeName = "bigint";
            break;
        case CHAR:
            typeName = String.format("char(%d)", extraTypeInfo.hiveCharMaxLength);
            break;
        case VARCHAR:
            typeName = String.format("varchar(%d)", extraTypeInfo.hiveVarcharMaxLength);
            break;
        case DECIMAL:
            typeName = String.format("decimal(%d,%d)", extraTypeInfo.precision, extraTypeInfo.scale);
            break;
        default:
            // No type name difference or adornment.
            typeName = primitiveCategory.name().toLowerCase();
            break;
    }
    PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(typeName);
    return primitiveTypeInfo;
}

Example 5 with ExtraTypeInfo

use of org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo in project hive by apache.

the class TestBinarySortableSerDe method testBinarySortableSerDe.

public void testBinarySortableSerDe() throws Throwable {
    try {
        System.out.println("Beginning Test testBinarySortableSerDe:");
        int num = 1000;
        Random r = new Random(1234);
        MyTestClass[] rows = new MyTestClass[num];
        int i;
        // First try non-random values
        for (i = 0; i < MyTestClass.nrDecimal.length; i++) {
            MyTestClass t = new MyTestClass();
            t.nonRandomFill(i);
            rows[i] = t;
        }
        for (; i < num; i++) {
            MyTestClass t = new MyTestClass();
            ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
            t.randomFill(r, extraTypeInfo);
            rows[i] = t;
        }
        StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
        sort(rows, rowOI);
        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
        String order;
        order = StringUtils.leftPad("", MyTestClass.fieldCount, '+');
        String nullOrder;
        nullOrder = StringUtils.leftPad("", MyTestClass.fieldCount, 'a');
        testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, order, nullOrder), true);
        order = StringUtils.leftPad("", MyTestClass.fieldCount, '-');
        nullOrder = StringUtils.leftPad("", MyTestClass.fieldCount, 'z');
        testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, order, nullOrder), false);
        System.out.println("Test testTBinarySortableProtocol passed!");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}