Examples with TypeInfoUtils.getTypeInfoFromTypeString org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString used on opensource projects

Example 41 with TypeInfoUtils.getTypeInfoFromTypeString

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString in project cdap by caskdata.

the class StandardObjectInspectorsTest method testStandardUnionObjectInspector.

@Test
public void testStandardUnionObjectInspector() throws Throwable {
    try {
        ArrayList<ObjectInspector> objectInspectors = new ArrayList<>();
        // add primitive types
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
        // add a list
        objectInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector));
        // add a map
        objectInspectors.add(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector));
        // add a struct
        List<String> fieldNames = new ArrayList<>();
        fieldNames.add("myDouble");
        fieldNames.add("myLong");
        ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<>();
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
        objectInspectors.add(ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors));
        StandardUnionObjectInspector uoi1 = ObjectInspectorFactory.getStandardUnionObjectInspector(objectInspectors);
        StandardUnionObjectInspector uoi2 = ObjectInspectorFactory.getStandardUnionObjectInspector((ArrayList<ObjectInspector>) objectInspectors.clone());
        Assert.assertEquals(uoi1, uoi2);
        Assert.assertEquals(ObjectInspectorUtils.getObjectInspectorName(uoi1), ObjectInspectorUtils.getObjectInspectorName(uoi2));
        Assert.assertTrue(ObjectInspectorUtils.compareTypes(uoi1, uoi2));
        // compareSupported returns false because Union can contain
        // an object of Map
        Assert.assertFalse(ObjectInspectorUtils.compareSupported(uoi1));
        // construct unionObjectInspector without Map field.
        ArrayList<ObjectInspector> ois = (ArrayList<ObjectInspector>) objectInspectors.clone();
        ois.set(4, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        Assert.assertTrue(ObjectInspectorUtils.compareSupported(ObjectInspectorFactory.getStandardUnionObjectInspector(ois)));
        // metadata
        Assert.assertEquals(Category.UNION, uoi1.getCategory());
        List<? extends ObjectInspector> uois = uoi1.getObjectInspectors();
        Assert.assertEquals(6, uois.size());
        for (int i = 0; i < 6; i++) {
            Assert.assertEquals(objectInspectors.get(i), uois.get(i));
        }
        StringBuilder unionTypeName = new StringBuilder();
        unionTypeName.append("uniontype<");
        for (int i = 0; i < uois.size(); i++) {
            if (i > 0) {
                unionTypeName.append(",");
            }
            unionTypeName.append(uois.get(i).getTypeName());
        }
        unionTypeName.append(">");
        Assert.assertEquals(unionTypeName.toString(), uoi1.getTypeName());
        // TypeInfo
        TypeInfo typeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi1);
        Assert.assertEquals(Category.UNION, typeInfo1.getCategory());
        Assert.assertEquals(UnionTypeInfo.class.getName(), typeInfo1.getClass().getName());
        Assert.assertEquals(typeInfo1.getTypeName(), uoi1.getTypeName());
        Assert.assertEquals(typeInfo1, TypeInfoUtils.getTypeInfoFromTypeString(uoi1.getTypeName()));
        TypeInfo typeInfo2 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi2);
        Assert.assertEquals(typeInfo1, typeInfo2);
        Assert.assertEquals(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo2));
        Assert.assertEquals(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo2));
        // null
        Assert.assertNull(uoi1.getField(null));
        Assert.assertEquals(-1, uoi1.getTag(null));
        // Union
        UnionObject union = new StandardUnionObjectInspector.StandardUnion((byte) 0, 1);
        Assert.assertEquals(0, uoi1.getTag(union));
        Assert.assertEquals(1, uoi1.getField(union));
        Assert.assertEquals("{0:1}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 0, 1), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(1));
        union = new StandardUnionObjectInspector.StandardUnion((byte) 1, "two");
        Assert.assertEquals(1, uoi1.getTag(union));
        Assert.assertEquals("two", uoi1.getField(union));
        Assert.assertEquals("{1:\"two\"}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 1, "two"), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals("two"));
        union = new StandardUnionObjectInspector.StandardUnion((byte) 2, true);
        Assert.assertEquals(2, uoi1.getTag(union));
        Assert.assertEquals(true, uoi1.getField(union));
        Assert.assertEquals("{2:true}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 2, true), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(true));
        ArrayList<Integer> iList = new ArrayList<>();
        iList.add(4);
        iList.add(5);
        union = new StandardUnionObjectInspector.StandardUnion((byte) 3, iList);
        Assert.assertEquals(3, uoi1.getTag(union));
        Assert.assertEquals(iList, uoi1.getField(union));
        Assert.assertEquals("{3:[4,5]}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 3, iList.clone()), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(iList));
        HashMap<Integer, String> map = new HashMap<>();
        map.put(6, "six");
        map.put(7, "seven");
        map.put(8, "eight");
        union = new StandardUnionObjectInspector.StandardUnion((byte) 4, map);
        Assert.assertEquals(4, uoi1.getTag(union));
        Assert.assertEquals(map, uoi1.getField(union));
        Assert.assertEquals("{4:{6:\"six\",7:\"seven\",8:\"eight\"}}", SerDeUtils.getJSONString(union, uoi1));
        Throwable th = null;
        try {
            ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 4, map.clone()), uoi2, null);
        } catch (Throwable t) {
            th = t;
        }
        Assert.assertNotNull(th);
        Assert.assertEquals("Compare on map type not supported!", th.getMessage());
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(map));
        ArrayList<Object> struct = new ArrayList<>(2);
        struct.add(9.0);
        struct.add(10L);
        union = new StandardUnionObjectInspector.StandardUnion((byte) 5, struct);
        Assert.assertEquals(5, uoi1.getTag(union));
        Assert.assertEquals(struct, uoi1.getField(union));
        Assert.assertEquals("{5:{\"mydouble\":9.0,\"mylong\":10}}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 5, struct.clone()), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(struct));
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 42 with TypeInfoUtils.getTypeInfoFromTypeString

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString in project hive by apache.

the class HCatSchemaUtils method getHCatFieldSchema.

/**
 * Convert a HCatFieldSchema to a FieldSchema
 * @param fs FieldSchema to convert
 * @return HCatFieldSchema representation of FieldSchema
 * @throws HCatException
 */
public static HCatFieldSchema getHCatFieldSchema(FieldSchema fs) throws HCatException {
    String fieldName = fs.getName();
    TypeInfo baseTypeInfo = TypeInfoUtils.getTypeInfoFromTypeString(fs.getType());
    return getHCatFieldSchema(fieldName, baseTypeInfo, fs.getComment());
}

Example 43 with TypeInfoUtils.getTypeInfoFromTypeString

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString in project hive by apache.

the class SpecialCases method addSpecialCasesParametersToOutputJobProperties.

/**
 * Method to do any file-format specific special casing while
 * instantiating a storage handler to write. We set any parameters
 * we want to be visible to the job in jobProperties, and this will
 * be available to the job via jobconf at run time.
 *
 * This is mostly intended to be used by StorageHandlers that wrap
 * File-based OutputFormats such as FosterStorageHandler that wraps
 * RCFile, ORC, etc.
 *
 * @param jobProperties : map to write to
 * @param jobInfo : information about this output job to read from
 * @param ofclass : the output format in use
 */
public static void addSpecialCasesParametersToOutputJobProperties(Map<String, String> jobProperties, OutputJobInfo jobInfo, Class<? extends OutputFormat> ofclass) {
    if (ofclass == RCFileOutputFormat.class) {
        // RCFile specific parameter
        jobProperties.put(HiveConf.ConfVars.HIVE_RCFILE_COLUMN_NUMBER_CONF.varname, Integer.toOctalString(jobInfo.getOutputSchema().getFields().size()));
    } else if (ofclass == OrcOutputFormat.class) {
        // Special cases for ORC
        // We need to check table properties to see if a couple of parameters,
        // such as compression parameters are defined. If they are, then we copy
        // them to job properties, so that it will be available in jobconf at runtime
        // See HIVE-5504 for details
        Map<String, String> tableProps = jobInfo.getTableInfo().getTable().getParameters();
        for (OrcConf property : OrcConf.values()) {
            String propName = property.getAttribute();
            if (tableProps.containsKey(propName)) {
                jobProperties.put(propName, tableProps.get(propName));
            }
        }
    } else if (ofclass == AvroContainerOutputFormat.class) {
        // Special cases for Avro. As with ORC, we make table properties that
        // Avro is interested in available in jobconf at runtime
        Map<String, String> tableProps = jobInfo.getTableInfo().getTable().getParameters();
        for (AvroSerdeUtils.AvroTableProperties property : AvroSerdeUtils.AvroTableProperties.values()) {
            String propName = property.getPropName();
            if (tableProps.containsKey(propName)) {
                String propVal = tableProps.get(propName);
                jobProperties.put(propName, tableProps.get(propName));
            }
        }
        Properties properties = new Properties();
        properties.put("name", jobInfo.getTableName());
        List<String> colNames = jobInfo.getOutputSchema().getFieldNames();
        List<TypeInfo> colTypes = new ArrayList<TypeInfo>();
        for (HCatFieldSchema field : jobInfo.getOutputSchema().getFields()) {
            colTypes.add(TypeInfoUtils.getTypeInfoFromTypeString(field.getTypeString()));
        }
        if (jobProperties.get(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName()) == null || jobProperties.get(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName()).isEmpty()) {
            jobProperties.put(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), AvroSerDe.getSchemaFromCols(properties, colNames, colTypes, null).toString());
        }
    } else if (ofclass == MapredParquetOutputFormat.class) {
        // Handle table properties
        Properties tblProperties = new Properties();
        Map<String, String> tableProps = jobInfo.getTableInfo().getTable().getParameters();
        for (String key : tableProps.keySet()) {
            if (ParquetTableUtils.isParquetProperty(key)) {
                tblProperties.put(key, tableProps.get(key));
            }
        }
        // Handle table schema
        List<String> colNames = jobInfo.getOutputSchema().getFieldNames();
        List<TypeInfo> colTypes = new ArrayList<TypeInfo>();
        for (HCatFieldSchema field : jobInfo.getOutputSchema().getFields()) {
            colTypes.add(TypeInfoUtils.getTypeInfoFromTypeString(field.getTypeString()));
        }
        String parquetSchema = HiveSchemaConverter.convert(colNames, colTypes).toString();
        jobProperties.put(DataWritableWriteSupport.PARQUET_HIVE_SCHEMA, parquetSchema);
        jobProperties.putAll(Maps.fromProperties(tblProperties));
    }
}

Example 44 with TypeInfoUtils.getTypeInfoFromTypeString

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString in project hive by apache.

the class TestLazyHBaseObject method testLazyHBaseCellMap3.

/**
 * Test the LazyHBaseCellMap class for the case where both the key and the value in the family
 * map are stored in binary format using the appropriate LazyPrimitive objects.
 * @throws SerDeException
 */
public void testLazyHBaseCellMap3() throws SerDeException {
    Text nullSequence = new Text("\\N");
    TypeInfo mapBinaryIntKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<int,int>");
    ObjectInspector oi = LazyFactory.createLazyObjectInspector(mapBinaryIntKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    LazyHBaseCellMap hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    List<Cell> kvs = new ArrayList<Cell>();
    byte[] rowKey = "row-key".getBytes();
    byte[] cfInt = "cf-int".getBytes();
    kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(1), Bytes.toBytes(1)));
    Result result = Result.create(kvs);
    List<Boolean> mapBinaryStorage = new ArrayList<Boolean>();
    mapBinaryStorage.add(true);
    mapBinaryStorage.add(true);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    IntWritable expectedIntValue = new IntWritable(1);
    LazyPrimitive<?, ?> lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(Integer.MIN_VALUE), Bytes.toBytes(Integer.MIN_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    expectedIntValue = new IntWritable(Integer.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfInt, Bytes.toBytes(Integer.MAX_VALUE), Bytes.toBytes(Integer.MAX_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfInt, mapBinaryStorage);
    expectedIntValue = new IntWritable(Integer.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedIntValue);
    assertEquals(expectedIntValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryByteKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<tinyint,tinyint>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryByteKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfByte = "cf-byte".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte[] { (byte) 1 }, new byte[] { (byte) 1 }));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    ByteWritable expectedByteValue = new ByteWritable((byte) 1);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte[] { Byte.MIN_VALUE }, new byte[] { Byte.MIN_VALUE }));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    expectedByteValue = new ByteWritable(Byte.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfByte, new byte[] { Byte.MAX_VALUE }, new byte[] { Byte.MAX_VALUE }));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfByte, mapBinaryStorage);
    expectedByteValue = new ByteWritable(Byte.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedByteValue);
    assertEquals(expectedByteValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryShortKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<smallint,smallint>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryShortKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfShort = "cf-short".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes((short) 1), Bytes.toBytes((short) 1)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    ShortWritable expectedShortValue = new ShortWritable((short) 1);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MIN_VALUE), Bytes.toBytes(Short.MIN_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    expectedShortValue = new ShortWritable(Short.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfShort, Bytes.toBytes(Short.MAX_VALUE), Bytes.toBytes(Short.MAX_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfShort, mapBinaryStorage);
    expectedShortValue = new ShortWritable(Short.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedShortValue);
    assertEquals(expectedShortValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryLongKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<bigint,bigint>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryLongKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfLong = "cf-long".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes((long) 1), Bytes.toBytes((long) 1)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    LongWritable expectedLongValue = new LongWritable(1);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MIN_VALUE), Bytes.toBytes(Long.MIN_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    expectedLongValue = new LongWritable(Long.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfLong, Bytes.toBytes(Long.MAX_VALUE), Bytes.toBytes(Long.MAX_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfLong, mapBinaryStorage);
    expectedLongValue = new LongWritable(Long.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedLongValue);
    assertEquals(expectedLongValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryFloatKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<float,float>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryFloatKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfFloat = "cf-float".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) 1.0F), Bytes.toBytes((float) 1.0F)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    FloatWritable expectedFloatValue = new FloatWritable(1.0F);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MIN_VALUE), Bytes.toBytes((float) Float.MIN_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    expectedFloatValue = new FloatWritable(Float.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfFloat, Bytes.toBytes((float) Float.MAX_VALUE), Bytes.toBytes((float) Float.MAX_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfFloat, mapBinaryStorage);
    expectedFloatValue = new FloatWritable(Float.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedFloatValue);
    assertEquals(expectedFloatValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryDoubleKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<double,double>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryDoubleKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfDouble = "cf-double".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(1.0), Bytes.toBytes(1.0)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    DoubleWritable expectedDoubleValue = new DoubleWritable(1.0);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MIN_VALUE), Bytes.toBytes(Double.MIN_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    expectedDoubleValue = new DoubleWritable(Double.MIN_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfDouble, Bytes.toBytes(Double.MAX_VALUE), Bytes.toBytes(Double.MAX_VALUE)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfDouble, mapBinaryStorage);
    expectedDoubleValue = new DoubleWritable(Double.MAX_VALUE);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedDoubleValue);
    assertEquals(expectedDoubleValue, lazyPrimitive.getWritableObject());
    TypeInfo mapBinaryBooleanKeyValue = TypeInfoUtils.getTypeInfoFromTypeString("map<boolean,boolean>");
    oi = LazyFactory.createLazyObjectInspector(mapBinaryBooleanKeyValue, new byte[] { (byte) 1, (byte) 2 }, 0, nullSequence, false, (byte) 0);
    hbaseCellMap = new LazyHBaseCellMap((LazyMapObjectInspector) oi);
    byte[] cfBoolean = "cf-boolean".getBytes();
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(false), Bytes.toBytes(false)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
    BooleanWritable expectedBooleanValue = new BooleanWritable(false);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);
    assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());
    kvs.clear();
    kvs.add(new KeyValue(rowKey, cfBoolean, Bytes.toBytes(true), Bytes.toBytes(true)));
    result = Result.create(kvs);
    hbaseCellMap.init(result, cfBoolean, mapBinaryStorage);
    expectedBooleanValue = new BooleanWritable(true);
    lazyPrimitive = (LazyPrimitive<?, ?>) hbaseCellMap.getMapValueElement(expectedBooleanValue);
    assertEquals(expectedBooleanValue, lazyPrimitive.getWritableObject());
}

Example 45 with TypeInfoUtils.getTypeInfoFromTypeString

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils.getTypeInfoFromTypeString in project hive by apache.

the class HCatUtil method validatePartitionSchema.

/**
 * Validate partition schema, checks if the column types match between the
 * partition and the existing table schema. Returns the list of columns
 * present in the partition but not in the table.
 *
 * @param table the table
 * @param partitionSchema the partition schema
 * @return the list of newly added fields
 * @throws IOException Signals that an I/O exception has occurred.
 */
public static List<FieldSchema> validatePartitionSchema(Table table, HCatSchema partitionSchema) throws IOException {
    Map<String, FieldSchema> partitionKeyMap = new HashMap<String, FieldSchema>();
    for (FieldSchema field : table.getPartitionKeys()) {
        partitionKeyMap.put(field.getName().toLowerCase(), field);
    }
    List<FieldSchema> tableCols = table.getCols();
    List<FieldSchema> newFields = new ArrayList<FieldSchema>();
    for (int i = 0; i < partitionSchema.getFields().size(); i++) {
        FieldSchema field = HCatSchemaUtils.getFieldSchema(partitionSchema.getFields().get(i));
        FieldSchema tableField;
        if (i < tableCols.size()) {
            tableField = tableCols.get(i);
            if (!tableField.getName().equalsIgnoreCase(field.getName())) {
                throw new HCatException(ErrorType.ERROR_SCHEMA_COLUMN_MISMATCH, "Expected column <" + tableField.getName() + "> at position " + (i + 1) + ", found column <" + field.getName() + ">");
            }
        } else {
            tableField = partitionKeyMap.get(field.getName().toLowerCase());
            if (tableField != null) {
                throw new HCatException(ErrorType.ERROR_SCHEMA_PARTITION_KEY, "Key <" + field.getName() + ">");
            }
        }
        if (tableField == null) {
            // field present in partition but not in table
            newFields.add(field);
        } else {
            // field present in both. validate type has not changed
            TypeInfo partitionType = TypeInfoUtils.getTypeInfoFromTypeString(field.getType());
            TypeInfo tableType = TypeInfoUtils.getTypeInfoFromTypeString(tableField.getType());
            if (!partitionType.equals(tableType)) {
                String msg = "Column <" + field.getName() + ">, expected <" + tableType.getTypeName() + ">, got <" + partitionType.getTypeName() + ">";
                LOG.warn(msg);
                throw new HCatException(ErrorType.ERROR_SCHEMA_TYPE_MISMATCH, msg);
            }
        }
    }
    return newFields;
}