Examples with PrimitiveObjectInspectorFactory.javaStringObjectInspector org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector used on opensource projects

Example 36 with PrimitiveObjectInspectorFactory.javaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector in project hive by apache.

the class RegexSerDe method initialize.

@Override
public void initialize(Configuration conf, Properties tbl) throws SerDeException {
    // We can get the table definition from tbl.
    // Read the configuration parameters
    inputRegex = tbl.getProperty(INPUT_REGEX);
    outputFormatString = tbl.getProperty(OUTPUT_FORMAT_STRING);
    String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
    String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
    boolean inputRegexIgnoreCase = "true".equalsIgnoreCase(tbl.getProperty(INPUT_REGEX_CASE_SENSITIVE));
    // Parse the configuration parameters
    if (inputRegex != null) {
        inputPattern = Pattern.compile(inputRegex, Pattern.DOTALL + (inputRegexIgnoreCase ? Pattern.CASE_INSENSITIVE : 0));
    } else {
        inputPattern = null;
    }
    final String columnNameDelimiter = tbl.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tbl.getProperty(serdeConstants.COLUMN_NAME_DELIMITER) : String.valueOf(SerDeUtils.COMMA);
    List<String> columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
    List<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    assert columnNames.size() == columnTypes.size();
    numColumns = columnNames.size();
    // All columns have to be of type STRING.
    for (int c = 0; c < numColumns; c++) {
        if (!columnTypes.get(c).equals(TypeInfoFactory.stringTypeInfo)) {
            throw new SerDeException(getClass().getName() + " only accepts string columns, but column[" + c + "] named " + columnNames.get(c) + " has type " + columnTypes.get(c));
        }
    }
    // Constructing the row ObjectInspector:
    // The row consists of some string columns, each column will be a java
    // String object.
    List<ObjectInspector> columnOIs = new ArrayList<ObjectInspector>(columnNames.size());
    for (int c = 0; c < numColumns; c++) {
        columnOIs.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    }
    // StandardStruct uses ArrayList to store the row.
    rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, columnOIs);
    // Constructing the row object, etc, which will be reused for all rows.
    row = new ArrayList<String>(numColumns);
    for (int c = 0; c < numColumns; c++) {
        row.add(null);
    }
    outputFields = new Object[numColumns];
    outputRowText = new Text();
}

Example 37 with PrimitiveObjectInspectorFactory.javaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector in project hive by apache.

the class HBaseUtils method deserializePartitionKey.

// Deserialize a partition key and return _only_ the partition values.
private static List<String> deserializePartitionKey(List<FieldSchema> partitions, byte[] key, Configuration conf) {
    StringBuffer names = new StringBuffer();
    names.append("dbName,tableName,");
    StringBuffer types = new StringBuffer();
    types.append("string,string,");
    for (int i = 0; i < partitions.size(); i++) {
        names.append(partitions.get(i).getName());
        types.append(TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType()));
        if (i != partitions.size() - 1) {
            names.append(",");
            types.append(",");
        }
    }
    BinarySortableSerDe serDe = new BinarySortableSerDe();
    Properties props = new Properties();
    props.setProperty(serdeConstants.LIST_COLUMNS, names.toString());
    props.setProperty(serdeConstants.LIST_COLUMN_TYPES, types.toString());
    try {
        serDe.initialize(conf, props);
        List deserializedkeys = ((List) serDe.deserialize(new BytesWritable(key))).subList(2, partitions.size() + 2);
        List<String> partitionKeys = new ArrayList<String>();
        for (int i = 0; i < deserializedkeys.size(); i++) {
            Object deserializedKey = deserializedkeys.get(i);
            if (deserializedKey == null) {
                partitionKeys.add(HiveConf.getVar(conf, HiveConf.ConfVars.DEFAULTPARTITIONNAME));
            } else {
                TypeInfo inputType = TypeInfoUtils.getTypeInfoFromTypeString(partitions.get(i).getType());
                ObjectInspector inputOI = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(inputType);
                Converter converter = ObjectInspectorConverters.getConverter(inputOI, PrimitiveObjectInspectorFactory.javaStringObjectInspector);
                partitionKeys.add((String) converter.convert(deserializedKey));
            }
        }
        return partitionKeys;
    } catch (SerDeException e) {
        throw new RuntimeException("Error when deserialize key", e);
    }
}

Example 38 with PrimitiveObjectInspectorFactory.javaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector in project hive by apache.

the class TestOperators method setUp.

@Override
protected void setUp() {
    r = new InspectableObject[5];
    ArrayList<String> names = new ArrayList<String>(3);
    names.add("col0");
    names.add("col1");
    names.add("col2");
    ArrayList<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>(3);
    objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    for (int i = 0; i < 5; i++) {
        ArrayList<String> data = new ArrayList<String>();
        data.add("" + i);
        data.add("" + (i + 1));
        data.add("" + (i + 2));
        try {
            r[i] = new InspectableObject();
            r[i].o = data;
            r[i].oi = ObjectInspectorFactory.getStandardStructObjectInspector(names, objectInspectors);
        } catch (Throwable e) {
            throw new RuntimeException(e);
        }
    }
}

Example 39 with PrimitiveObjectInspectorFactory.javaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector in project hive by apache.

the class TestStandardObjectInspectors method testStandardUnionObjectInspector.

@SuppressWarnings("unchecked")
public void testStandardUnionObjectInspector() throws Throwable {
    try {
        ArrayList<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>();
        // add primitive types
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
        // add a list
        objectInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector));
        // add a map
        objectInspectors.add(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector));
        // add a struct
        List<String> fieldNames = new ArrayList<String>();
        fieldNames.add("myDouble");
        fieldNames.add("myLong");
        ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
        objectInspectors.add(ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors));
        StandardUnionObjectInspector uoi1 = ObjectInspectorFactory.getStandardUnionObjectInspector(objectInspectors);
        StandardUnionObjectInspector uoi2 = ObjectInspectorFactory.getStandardUnionObjectInspector((ArrayList<ObjectInspector>) objectInspectors.clone());
        assertEquals(uoi1, uoi2);
        assertEquals(ObjectInspectorUtils.getObjectInspectorName(uoi1), ObjectInspectorUtils.getObjectInspectorName(uoi2));
        assertTrue(ObjectInspectorUtils.compareTypes(uoi1, uoi2));
        // compareSupported returns false because Union can contain
        // an object of Map
        assertFalse(ObjectInspectorUtils.compareSupported(uoi1));
        // construct unionObjectInspector without Map field.
        ArrayList<ObjectInspector> ois = (ArrayList<ObjectInspector>) objectInspectors.clone();
        ois.set(4, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        assertTrue(ObjectInspectorUtils.compareSupported(ObjectInspectorFactory.getStandardUnionObjectInspector(ois)));
        // metadata
        assertEquals(Category.UNION, uoi1.getCategory());
        List<? extends ObjectInspector> uois = uoi1.getObjectInspectors();
        assertEquals(6, uois.size());
        for (int i = 0; i < 6; i++) {
            assertEquals(objectInspectors.get(i), uois.get(i));
        }
        StringBuilder unionTypeName = new StringBuilder();
        unionTypeName.append("uniontype<");
        for (int i = 0; i < uois.size(); i++) {
            if (i > 0) {
                unionTypeName.append(",");
            }
            unionTypeName.append(uois.get(i).getTypeName());
        }
        unionTypeName.append(">");
        assertEquals(unionTypeName.toString(), uoi1.getTypeName());
        // TypeInfo
        TypeInfo typeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi1);
        assertEquals(Category.UNION, typeInfo1.getCategory());
        assertEquals(UnionTypeInfo.class.getName(), typeInfo1.getClass().getName());
        assertEquals(typeInfo1.getTypeName(), uoi1.getTypeName());
        assertEquals(typeInfo1, TypeInfoUtils.getTypeInfoFromTypeString(uoi1.getTypeName()));
        TypeInfo typeInfo2 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi2);
        assertEquals(typeInfo1, typeInfo2);
        assertEquals(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo2));
        assertEquals(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo2));
        // null
        assertNull(uoi1.getField(null));
        assertEquals(-1, uoi1.getTag(null));
        // Union
        UnionObject union = new StandardUnion((byte) 0, 1);
        assertEquals(0, uoi1.getTag(union));
        assertEquals(1, uoi1.getField(union));
        assertEquals("{0:1}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 0, 1), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(1));
        union = new StandardUnion((byte) 1, "two");
        assertEquals(1, uoi1.getTag(union));
        assertEquals("two", uoi1.getField(union));
        assertEquals("{1:\"two\"}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 1, "two"), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals("two"));
        union = new StandardUnion((byte) 2, true);
        assertEquals(2, uoi1.getTag(union));
        assertEquals(true, uoi1.getField(union));
        assertEquals("{2:true}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 2, true), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(true));
        ArrayList<Integer> iList = new ArrayList<Integer>();
        iList.add(4);
        iList.add(5);
        union = new StandardUnion((byte) 3, iList);
        assertEquals(3, uoi1.getTag(union));
        assertEquals(iList, uoi1.getField(union));
        assertEquals("{3:[4,5]}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 3, iList.clone()), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(iList));
        HashMap<Integer, String> map = new HashMap<Integer, String>();
        map.put(6, "six");
        map.put(7, "seven");
        map.put(8, "eight");
        union = new StandardUnion((byte) 4, map);
        assertEquals(4, uoi1.getTag(union));
        assertEquals(map, uoi1.getField(union));
        assertEquals("{4:{6:\"six\",7:\"seven\",8:\"eight\"}}", SerDeUtils.getJSONString(union, uoi1));
        Throwable th = null;
        try {
            ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 4, map.clone()), uoi2, null);
        } catch (Throwable t) {
            th = t;
        }
        assertNotNull(th);
        assertEquals("Compare on map type not supported!", th.getMessage());
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(map));
        ArrayList<Object> struct = new ArrayList<Object>(2);
        struct.add(9.0);
        struct.add(10L);
        union = new StandardUnion((byte) 5, struct);
        assertEquals(5, uoi1.getTag(union));
        assertEquals(struct, uoi1.getField(union));
        assertEquals("{5:{\"mydouble\":9.0,\"mylong\":10}}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 5, struct.clone()), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(struct));
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 40 with PrimitiveObjectInspectorFactory.javaStringObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory.javaStringObjectInspector in project hive by apache.

the class TestThriftObjectInspectors method testThriftSetObjectInspector.

@SuppressWarnings("unchecked")
public void testThriftSetObjectInspector() throws Throwable {
    try {
        ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(SetIntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(SetIntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        assertEquals(oi1, oi2);
        // metadata
        assertEquals(Category.STRUCT, oi1.getCategory());
        StructObjectInspector soi = (StructObjectInspector) oi1;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        assertEquals(2, fields.size());
        assertEquals(fields.get(0), soi.getStructFieldRef("sIntString"));
        assertEquals(fields.get(1), soi.getStructFieldRef("aString"));
        // null
        for (int i = 0; i < fields.size(); i++) {
            assertNull(soi.getStructFieldData(null, fields.get(i)));
        }
        // real object
        IntString s1 = new IntString();
        s1.setMyint(1);
        s1.setMyString("test");
        s1.setUnderscore_int(2);
        Set<IntString> set1 = new HashSet<IntString>();
        set1.add(s1);
        SetIntString s = new SetIntString();
        s.setSIntString(set1);
        s.setAString("setString");
        assertEquals(set1, soi.getStructFieldData(s, fields.get(0)));
        assertEquals("setString", soi.getStructFieldData(s, fields.get(1)));
        // sub fields
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)), fields.get(0).getFieldObjectInspector());
        assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
        // compare set fields
        ListObjectInspector loi = (ListObjectInspector) fields.get(0).getFieldObjectInspector();
        assertEquals(1, loi.getListLength(set1));
        List<IntString> list = (List<IntString>) loi.getList(set1);
        assertEquals(1, list.size());
        s1 = (IntString) loi.getListElement(list, 0);
        assertEquals(1, s1.getMyint());
        assertEquals("test", s1.getMyString());
        assertEquals(2, s1.getUnderscore_int());
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}