Examples with ObjectInspectorFactory.getReflectionObjectInspector org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector used on opensource projects

Example 66 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestBinarySortableSerDe method testBinarySortableSerDe.

public void testBinarySortableSerDe() throws Throwable {
    try {
        System.out.println("Beginning Test testBinarySortableSerDe:");
        int num = 1000;
        Random r = new Random(1234);
        MyTestClass[] rows = new MyTestClass[num];
        int i;
        // First try non-random values
        for (i = 0; i < MyTestClass.nrDecimal.length; i++) {
            MyTestClass t = new MyTestClass();
            t.nonRandomFill(i);
            rows[i] = t;
        }
        for (; i < num; i++) {
            MyTestClass t = new MyTestClass();
            ExtraTypeInfo extraTypeInfo = new ExtraTypeInfo();
            t.randomFill(r, extraTypeInfo);
            rows[i] = t;
        }
        StructObjectInspector rowOI = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(MyTestClass.class, ObjectInspectorOptions.JAVA);
        sort(rows, rowOI);
        String fieldNames = ObjectInspectorUtils.getFieldNames(rowOI);
        String fieldTypes = ObjectInspectorUtils.getFieldTypes(rowOI);
        String order;
        order = StringUtils.leftPad("", MyTestClass.fieldCount, '+');
        String nullOrder;
        nullOrder = StringUtils.leftPad("", MyTestClass.fieldCount, 'a');
        testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, order, nullOrder), true);
        order = StringUtils.leftPad("", MyTestClass.fieldCount, '-');
        nullOrder = StringUtils.leftPad("", MyTestClass.fieldCount, 'z');
        testBinarySortableSerDe(rows, rowOI, getSerDe(fieldNames, fieldTypes, order, nullOrder), false);
        System.out.println("Test testTBinarySortableProtocol passed!");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 67 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestLazyBinaryColumnarSerDe method testSerDeEmpties.

public void testSerDeEmpties() throws SerDeException {
    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
    String cols = ObjectInspectorUtils.getFieldNames(oi);
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, cols);
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
    OuterStruct outerStruct = new OuterStruct();
    outerStruct.mByte = 101;
    outerStruct.mShort = 2002;
    outerStruct.mInt = 3003;
    outerStruct.mLong = 4004l;
    outerStruct.mFloat = 5005.01f;
    outerStruct.mDouble = 6006.001d;
    outerStruct.mString = "";
    outerStruct.mBA = new byte[] { 'a' };
    outerStruct.mArray = new ArrayList<InnerStruct>();
    outerStruct.mMap = new TreeMap<String, InnerStruct>();
    outerStruct.mStruct = new InnerStruct(180018, 190019l);
    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
    ObjectInspector out_oi = serde.getObjectInspector();
    Object out_o = serde.deserialize(braw);
    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new SimpleMapEqualComparer())) {
        System.out.println("expected = " + SerDeUtils.getJSONString(outerStruct, oi));
        System.out.println("actual = " + SerDeUtils.getJSONString(out_o, out_oi));
        fail("Deserialized object does not compare");
    }
}

Example 68 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project hive by apache.

the class TestLazyBinaryColumnarSerDe method testSerDe.

public void testSerDe() throws SerDeException {
    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
    String cols = ObjectInspectorUtils.getFieldNames(oi);
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, cols);
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
    SerDeUtils.initializeSerDe(serde, new Configuration(), props, null);
    OuterStruct outerStruct = new OuterStruct();
    outerStruct.mByte = 1;
    outerStruct.mShort = 2;
    outerStruct.mInt = 3;
    outerStruct.mLong = 4l;
    outerStruct.mFloat = 5.01f;
    outerStruct.mDouble = 6.001d;
    outerStruct.mString = "seven";
    outerStruct.mBA = new byte[] { '2' };
    InnerStruct is1 = new InnerStruct(8, 9l);
    InnerStruct is2 = new InnerStruct(10, 11l);
    outerStruct.mArray = new ArrayList<InnerStruct>(2);
    outerStruct.mArray.add(is1);
    outerStruct.mArray.add(is2);
    outerStruct.mMap = new TreeMap<String, InnerStruct>();
    outerStruct.mMap.put(new String("twelve"), new InnerStruct(13, 14l));
    outerStruct.mMap.put(new String("fifteen"), new InnerStruct(16, 17l));
    outerStruct.mStruct = new InnerStruct(18, 19l);
    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
    ObjectInspector out_oi = serde.getObjectInspector();
    Object out_o = serde.deserialize(braw);
    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new CrossMapEqualComparer())) {
        System.out.println("expected = " + SerDeUtils.getJSONString(outerStruct, oi));
        System.out.println("actual = " + SerDeUtils.getJSONString(out_o, out_oi));
        fail("Deserialized object does not compare");
    }
}

Example 69 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project DataX by alibaba.

the class HdfsHelper method getColumnTypeInspectors.

/**
     * 根据writer配置的字段类型，构建inspector
     * @param columns
     * @return
     */
public List<ObjectInspector> getColumnTypeInspectors(List<Configuration> columns) {
    List<ObjectInspector> columnTypeInspectors = Lists.newArrayList();
    for (Configuration eachColumnConf : columns) {
        SupportHiveDataType columnType = SupportHiveDataType.valueOf(eachColumnConf.getString(Key.TYPE).toUpperCase());
        ObjectInspector objectInspector = null;
        switch(columnType) {
            case TINYINT:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(Byte.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case SMALLINT:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(Short.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case INT:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(Integer.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case BIGINT:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(Long.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case FLOAT:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(Float.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case DOUBLE:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(Double.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case TIMESTAMP:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(java.sql.Timestamp.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case DATE:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(java.sql.Date.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case STRING:
            case VARCHAR:
            case CHAR:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(String.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            case BOOLEAN:
                objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(Boolean.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
                break;
            default:
                throw DataXException.asDataXException(HdfsWriterErrorCode.ILLEGAL_VALUE, String.format("您的配置文件中的列配置信息有误. 因为DataX 不支持数据库写入这种字段类型. 字段名:[%s], 字段类型:[%d]. 请修改表中该字段的类型或者不同步该字段.", eachColumnConf.getString(Key.NAME), eachColumnConf.getString(Key.TYPE)));
        }
        columnTypeInspectors.add(objectInspector);
    }
    return columnTypeInspectors;
}

Example 70 with ObjectInspectorFactory.getReflectionObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getReflectionObjectInspector in project cdap by caskdata.

the class ExploreTableManager method hiveSchemaFor.

// TODO: replace with SchemaConverter.toHiveSchema when we tackle queries on Tables.
//       but unfortunately, SchemaConverter is not compatible with this, for example:
//       - a byte becomes a tinyint here, but an int there
//       - SchemaConverter sort fields alphabetically, whereas this preserves the order
//       - ExploreExtensiveSchemaTableTestRun will fail because of this
private String hiveSchemaFor(Type type) throws UnsupportedTypeException {
    // This call will make sure that the type is not recursive
    try {
        new ReflectionSchemaGenerator().generate(type, false);
    } catch (Exception e) {
        throw new UnsupportedTypeException("Unable to derive schema from " + type, e);
    }
    ObjectInspector objectInspector = ObjectInspectorFactory.getReflectionObjectInspector(type);
    if (!(objectInspector instanceof StructObjectInspector)) {
        throw new UnsupportedTypeException(String.format("Type must be a RECORD, but is %s", type.getClass().getName()));
    }
    StructObjectInspector structObjectInspector = (StructObjectInspector) objectInspector;
    StringBuilder sb = new StringBuilder();
    boolean first = true;
    for (StructField structField : structObjectInspector.getAllStructFieldRefs()) {
        if (first) {
            first = false;
        } else {
            sb.append(", ");
        }
        ObjectInspector oi = structField.getFieldObjectInspector();
        String typeName;
        typeName = oi.getTypeName();
        if (shouldEscapeColumns) {
            // a literal backtick(`) is represented as a double backtick(``)
            sb.append('`').append(structField.getFieldName().replace("`", "``")).append('`');
        } else {
            sb.append(structField.getFieldName());
        }
        sb.append(" ").append(typeName);
    }
    return sb.toString();
}