Examples with TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo used on opensource projects

Example 51 with TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo in project hive by apache.

the class ThriftJDBCBinarySerDe method initialize.

@Override
public void initialize(Configuration configuration, Properties tableProperties, Properties partitionProperties) throws SerDeException {
    super.initialize(configuration, tableProperties, partitionProperties);
    try {
        protocol = new TCompactProtocol(new TIOStreamTransport(output));
    } catch (TTransportException e) {
        throw new SerDeException(e);
    }
    // Get column names
    MAX_BUFFERED_ROWS = HiveConf.getIntVar(configuration, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_RESULTSET_DEFAULT_FETCH_SIZE);
    LOG.info("ThriftJDBCBinarySerDe max number of buffered columns: " + MAX_BUFFERED_ROWS);
    String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS);
    String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
    final String columnNameDelimiter = properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? properties.getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
    if (columnNameProperty.length() == 0) {
        columnNames = new ArrayList<String>();
    } else {
        columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
    }
    if (columnTypeProperty.length() == 0) {
        columnTypes = new ArrayList<TypeInfo>();
    } else {
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    }
    rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    rowObjectInspector = (StructObjectInspector) TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(rowTypeInfo);
    initializeRowAndColumns();
    try {
        thriftFormatter.initialize(configuration, properties);
    } catch (Exception e) {
        throw new SerDeException(e);
    }
}

Example 52 with TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo in project hive by apache.

the class TeradataBinarySerde method initialize.

@Override
public void initialize(Configuration configuration, Properties tableProperties, Properties partitionProperties) throws SerDeException {
    super.initialize(configuration, tableProperties, partitionProperties);
    columnNames = Arrays.asList(properties.getProperty(serdeConstants.LIST_COLUMNS).split(","));
    String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
    LOG.debug(serdeConstants.LIST_COLUMN_TYPES + ": " + columnTypeProperty);
    if (columnTypeProperty.length() == 0) {
        columnTypes = new ArrayList<TypeInfo>();
    } else {
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    }
    assert columnNames.size() == columnTypes.size();
    numCols = columnNames.size();
    // get the configured teradata timestamp precision
    // you can configure to generate timestamp of different precision in the binary file generated by TPT/BTEQ
    timestampPrecision = Integer.parseInt(properties.getProperty(TD_TIMESTAMP_PRECISION, DEFAULT_TIMESTAMP_PRECISION));
    // get the configured teradata char charset
    // in TD, latin charset will have 2 bytes per char and unicode will have 3 bytes per char
    charCharset = properties.getProperty(TD_CHAR_SET, DEFAULT_CHAR_CHARSET);
    if (!CHARSET_TO_BYTE_NUM.containsKey(charCharset)) {
        throw new SerDeException(format("%s isn't supported in Teradata Char Charset %s", charCharset, CHARSET_TO_BYTE_NUM.keySet()));
    }
    // All columns have to be primitive.
    // Constructing the row ObjectInspector:
    List<ObjectInspector> columnOIs = new ArrayList<ObjectInspector>(numCols);
    for (int i = 0; i < numCols; i++) {
        if (columnTypes.get(i).getCategory() != ObjectInspector.Category.PRIMITIVE) {
            throw new SerDeException(getClass().getName() + " only accepts primitive columns, but column[" + i + "] named " + columnNames.get(i) + " has category " + columnTypes.get(i).getCategory());
        }
        columnOIs.add(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(columnTypes.get(i)));
    }
    rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs);
    // Constructing the row object and will be reused for all rows
    row = new ArrayList<Object>(numCols);
    for (int i = 0; i < numCols; i++) {
        row.add(null);
    }
    // Initialize vars related to Null Array which represents the null bitmap
    int byteNumForNullArray = (numCols / 8) + ((numCols % 8 == 0) ? 0 : 1);
    LOG.debug(format("The Null Bytes for each record will have %s bytes", byteNumForNullArray));
    inForNull = new byte[byteNumForNullArray];
    out = new TeradataBinaryDataOutputStream();
    serializeBytesWritable = new BytesWritable();
    outForNull = new byte[byteNumForNullArray];
}

Example 53 with TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo in project hive by apache.

the class TestStandardObjectInspectors method testStandardUnionObjectInspector.

@SuppressWarnings("unchecked")
@Test
public void testStandardUnionObjectInspector() throws Throwable {
    try {
        ArrayList<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>();
        // add primitive types
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
        // add a list
        objectInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector));
        // add a map
        objectInspectors.add(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector));
        // add a struct
        List<String> fieldNames = new ArrayList<String>();
        fieldNames.add("myDouble");
        fieldNames.add("myLong");
        ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
        objectInspectors.add(ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors));
        StandardUnionObjectInspector uoi1 = ObjectInspectorFactory.getStandardUnionObjectInspector(objectInspectors);
        StandardUnionObjectInspector uoi2 = ObjectInspectorFactory.getStandardUnionObjectInspector((ArrayList<ObjectInspector>) objectInspectors.clone());
        assertEquals(uoi1, uoi2);
        assertEquals(ObjectInspectorUtils.getObjectInspectorName(uoi1), ObjectInspectorUtils.getObjectInspectorName(uoi2));
        assertTrue(ObjectInspectorUtils.compareTypes(uoi1, uoi2));
        // compareSupported returns false because Union can contain
        // an object of Map
        assertFalse(ObjectInspectorUtils.compareSupported(uoi1));
        // construct unionObjectInspector without Map field.
        ArrayList<ObjectInspector> ois = (ArrayList<ObjectInspector>) objectInspectors.clone();
        ois.set(4, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        assertTrue(ObjectInspectorUtils.compareSupported(ObjectInspectorFactory.getStandardUnionObjectInspector(ois)));
        // metadata
        assertEquals(Category.UNION, uoi1.getCategory());
        List<? extends ObjectInspector> uois = uoi1.getObjectInspectors();
        assertEquals(6, uois.size());
        for (int i = 0; i < 6; i++) {
            assertEquals(objectInspectors.get(i), uois.get(i));
        }
        StringBuilder unionTypeName = new StringBuilder();
        unionTypeName.append("uniontype<");
        for (int i = 0; i < uois.size(); i++) {
            if (i > 0) {
                unionTypeName.append(",");
            }
            unionTypeName.append(uois.get(i).getTypeName());
        }
        unionTypeName.append(">");
        assertEquals(unionTypeName.toString(), uoi1.getTypeName());
        // TypeInfo
        TypeInfo typeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi1);
        assertEquals(Category.UNION, typeInfo1.getCategory());
        assertEquals(UnionTypeInfo.class.getName(), typeInfo1.getClass().getName());
        assertEquals(typeInfo1.getTypeName(), uoi1.getTypeName());
        assertEquals(typeInfo1, TypeInfoUtils.getTypeInfoFromTypeString(uoi1.getTypeName()));
        TypeInfo typeInfo2 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi2);
        assertEquals(typeInfo1, typeInfo2);
        assertEquals(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo2));
        assertEquals(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo2));
        // null
        assertNull(uoi1.getField(null));
        assertEquals(-1, uoi1.getTag(null));
        // Union
        UnionObject union = new StandardUnion((byte) 0, 1);
        assertEquals(0, uoi1.getTag(union));
        assertEquals(1, uoi1.getField(union));
        assertEquals("{0:1}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 0, 1), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        union = new StandardUnion((byte) 1, "two");
        assertEquals(1, uoi1.getTag(union));
        assertEquals("two", uoi1.getField(union));
        assertEquals("{1:\"two\"}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 1, "two"), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        union = new StandardUnion((byte) 2, true);
        assertEquals(2, uoi1.getTag(union));
        assertEquals(true, uoi1.getField(union));
        assertEquals("{2:true}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 2, true), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        ArrayList<Integer> iList = new ArrayList<Integer>();
        iList.add(4);
        iList.add(5);
        union = new StandardUnion((byte) 3, iList);
        assertEquals(3, uoi1.getTag(union));
        assertEquals(iList, uoi1.getField(union));
        assertEquals("{3:[4,5]}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 3, iList.clone()), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        HashMap<Integer, String> map = new LinkedHashMap<Integer, String>();
        map.put(6, "six");
        map.put(7, "seven");
        map.put(8, "eight");
        union = new StandardUnion((byte) 4, map);
        assertEquals(4, uoi1.getTag(union));
        assertEquals(map, uoi1.getField(union));
        assertEquals("{4:{6:\"six\",7:\"seven\",8:\"eight\"}}", SerDeUtils.getJSONString(union, uoi1));
        Throwable th = null;
        try {
            ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 4, map.clone()), uoi2, null);
        } catch (Throwable t) {
            th = t;
        }
        assertNotNull(th);
        assertEquals("Compare on map type not supported!", th.getMessage());
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        ArrayList<Object> struct = new ArrayList<Object>(2);
        struct.add(9.0);
        struct.add(10L);
        union = new StandardUnion((byte) 5, struct);
        assertEquals(5, uoi1.getTag(union));
        assertEquals(struct, uoi1.getField(union));
        assertEquals("{5:{\"mydouble\":9.0,\"mylong\":10}}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 5, struct.clone()), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 54 with TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo in project incubator-gobblin by apache.

the class OrcTestTools method writeAsOrcBinary.

/**
 * AvroRow version of writeAsOrcBinary
 */
private void writeAsOrcBinary(OrcRowIterator input, TypeInfo schema, Path outputPath) throws IOException {
    Configuration configuration = new Configuration();
    // Note that it doesn't support schema evolution at all.
    // If the schema in realRow is inconsistent with given schema, writing into disk
    // would run into failure.
    ObjectInspector oi = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(schema);
    OrcFile.WriterOptions options = OrcFile.writerOptions(configuration).inspector(oi);
    Writer writer = null;
    while (input.hasNext()) {
        AvroRow avroRow = (AvroRow) input.next();
        if (writer == null) {
            options.inspector(avroRow.getInspector());
            writer = OrcFile.createWriter(outputPath, options);
        }
        writer.addRow(avroRow.realRow);
    }
    if (writer != null) {
        writer.close();
    }
}