Examples with StructField org.apache.hadoop.hive.serde2.objectinspector.StructField used on opensource projects

Example 61 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class TestDruidSerDe method deserializeQueryResults.

@SuppressWarnings("unchecked")
private void deserializeQueryResults(DruidSerDe serDe, String queryType, String jsonQuery, byte[] resultString, Object[][] records) throws SerDeException, IOException, NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException, InterruptedException, NoSuchMethodException, InvocationTargetException {
    // Initialize
    HttpClient httpClient = mock(HttpClient.class);
    SettableFuture<InputStream> futureResult = SettableFuture.create();
    futureResult.set(new ByteArrayInputStream(resultString));
    when(httpClient.go(anyObject(), any(HttpResponseHandler.class))).thenReturn(futureResult);
    DruidQueryRecordReader<?, ?> reader = DruidQueryBasedInputFormat.getDruidQueryReader(queryType);
    final HiveDruidSplit split = new HiveDruidSplit(jsonQuery, new Path("empty"), new String[] { "testing_host" });
    reader.initialize(split, new Configuration(), DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.SMILE_MAPPER, httpClient);
    StructObjectInspector oi = (StructObjectInspector) serDe.getObjectInspector();
    List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();
    // Check mapred
    DruidWritable writable = new DruidWritable();
    int pos = 0;
    while (reader.next(NullWritable.get(), writable)) {
        List<Object> row = (List<Object>) serDe.deserialize(writable);
        Object[] expectedFieldsData = records[pos];
        assertEquals(expectedFieldsData.length, fieldRefs.size());
        for (int i = 0; i < fieldRefs.size(); i++) {
            assertEquals("Field " + i + " type", expectedFieldsData[i].getClass(), row.get(i).getClass());
            Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
            assertEquals("Field " + i, expectedFieldsData[i], fieldData);
        }
        pos++;
    }
    assertEquals(pos, records.length);
    // Check mapreduce path
    futureResult = SettableFuture.create();
    futureResult.set(new ByteArrayInputStream(resultString));
    when(httpClient.go(anyObject(), any(HttpResponseHandler.class))).thenReturn(futureResult);
    reader = DruidQueryBasedInputFormat.getDruidQueryReader(queryType);
    reader.initialize(split, new Configuration(), DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.SMILE_MAPPER, httpClient);
    pos = 0;
    while (reader.nextKeyValue()) {
        List<Object> row = (List<Object>) serDe.deserialize(reader.getCurrentValue());
        Object[] expectedFieldsData = records[pos];
        assertEquals(expectedFieldsData.length, fieldRefs.size());
        for (int i = 0; i < fieldRefs.size(); i++) {
            assertEquals("Field " + i + " type", expectedFieldsData[i].getClass(), row.get(i).getClass());
            Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
            assertEquals("Field " + i, expectedFieldsData[i], fieldData);
        }
        pos++;
    }
    assertEquals(pos, records.length);
}

Example 62 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class HBaseRowSerializer method serializeField.

private void serializeField(Object value, StructField field, ColumnMapping colMap, Put put) throws IOException {
    if (value == null) {
        // a null object, we do not serialize it
        return;
    }
    // Get the field objectInspector and the field object.
    ObjectInspector foi = field.getFieldObjectInspector();
    // If the field corresponds to a column family in HBase
    if (colMap.qualifierName == null) {
        MapObjectInspector moi = (MapObjectInspector) foi;
        Map<?, ?> map = moi.getMap(value);
        if (map == null) {
            return;
        }
        ObjectInspector koi = moi.getMapKeyObjectInspector();
        ObjectInspector voi = moi.getMapValueObjectInspector();
        for (Map.Entry<?, ?> entry : map.entrySet()) {
            // Get the Key
            // Map keys are required to be primitive and may be serialized in binary format
            byte[] columnQualifierBytes = serialize(entry.getKey(), koi, 3, colMap.binaryStorage.get(0));
            if (columnQualifierBytes == null) {
                continue;
            }
            // Map values may be serialized in binary format when they are primitive and binary
            // serialization is the option selected
            byte[] bytes = serialize(entry.getValue(), voi, 3, colMap.binaryStorage.get(1));
            if (bytes == null) {
                continue;
            }
            put.addColumn(colMap.familyNameBytes, columnQualifierBytes, bytes);
        }
    } else {
        byte[] bytes;
        // delimited way.
        if (!foi.getCategory().equals(ObjectInspector.Category.PRIMITIVE) && colMap.isCategory(ObjectInspector.Category.PRIMITIVE)) {
            // we always serialize the String type using the escaped algorithm for LazyString
            bytes = serialize(SerDeUtils.getJSONString(value, foi), PrimitiveObjectInspectorFactory.javaStringObjectInspector, 1, false);
        } else {
            // use the serialization option switch to write primitive values as either a variable
            // length UTF8 string or a fixed width bytes if serializing in binary format
            bytes = serialize(value, foi, 1, colMap.binaryStorage.get(0));
        }
        if (bytes == null) {
            return;
        }
        put.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes, bytes);
    }
}

Example 63 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class HBaseRowSerializer method serialize.

private boolean serialize(Object obj, ObjectInspector objInspector, int level, ByteStream.Output ss) throws IOException {
    switch(objInspector.getCategory()) {
        case PRIMITIVE:
            LazyUtils.writePrimitiveUTF8(ss, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape);
            return true;
        case LIST:
            char separator = (char) separators[level];
            ListObjectInspector loi = (ListObjectInspector) objInspector;
            List<?> list = loi.getList(obj);
            ObjectInspector eoi = loi.getListElementObjectInspector();
            if (list == null) {
                return false;
            } else {
                for (int i = 0; i < list.size(); i++) {
                    if (i > 0) {
                        ss.write(separator);
                    }
                    Object currentItem = list.get(i);
                    if (currentItem != null) {
                        serialize(currentItem, eoi, level + 1, ss);
                    }
                }
            }
            return true;
        case MAP:
            char sep = (char) separators[level];
            char keyValueSeparator = (char) separators[level + 1];
            MapObjectInspector moi = (MapObjectInspector) objInspector;
            ObjectInspector koi = moi.getMapKeyObjectInspector();
            ObjectInspector voi = moi.getMapValueObjectInspector();
            Map<?, ?> map = moi.getMap(obj);
            if (map == null) {
                return false;
            } else {
                boolean first = true;
                for (Map.Entry<?, ?> entry : map.entrySet()) {
                    if (first) {
                        first = false;
                    } else {
                        ss.write(sep);
                    }
                    serialize(entry.getKey(), koi, level + 2, ss);
                    Object currentValue = entry.getValue();
                    if (currentValue != null) {
                        ss.write(keyValueSeparator);
                        serialize(currentValue, voi, level + 2, ss);
                    }
                }
            }
            return true;
        case STRUCT:
            sep = (char) separators[level];
            StructObjectInspector soi = (StructObjectInspector) objInspector;
            List<? extends StructField> fields = soi.getAllStructFieldRefs();
            list = soi.getStructFieldsDataAsList(obj);
            if (list == null) {
                return false;
            } else {
                for (int i = 0; i < list.size(); i++) {
                    if (i > 0) {
                        ss.write(sep);
                    }
                    Object currentItem = list.get(i);
                    if (currentItem != null) {
                        serialize(currentItem, fields.get(i).getFieldObjectInspector(), level + 1, ss);
                    }
                }
            }
            return true;
        case UNION:
            // union type currently not totally supported. See HIVE-2390
            return false;
        default:
            throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
    }
}

Example 64 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class LazyBinarySerDe2 method createLBSerializer.

/**
 * Generate a LBSerializer for the given ObjectInspector
 * @param oi
 * @return
 */
LBSerializer createLBSerializer(ObjectInspector oi) {
    switch(oi.getCategory()) {
        case PRIMITIVE:
            PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
            return createPrimitiveLBSerializer(poi);
        case LIST:
            ListObjectInspector loi = (ListObjectInspector) oi;
            ObjectInspector eoi = loi.getListElementObjectInspector();
            return new LBListSerializer(createLBSerializer(eoi));
        case MAP:
            MapObjectInspector moi = (MapObjectInspector) oi;
            ObjectInspector koi = moi.getMapKeyObjectInspector();
            ObjectInspector voi = moi.getMapValueObjectInspector();
            return new LBMapSerializer(createLBSerializer(koi), createLBSerializer(voi));
        case STRUCT:
            StructObjectInspector soi = (StructObjectInspector) oi;
            List<? extends StructField> fields = soi.getAllStructFieldRefs();
            LBSerializer[] fieldSerializers = new LBSerializer[fields.size()];
            for (int idx = 0; idx < fieldSerializers.length; ++idx) {
                fieldSerializers[idx] = createLBSerializer(fields.get(idx).getFieldObjectInspector());
            }
            return new LBStructSerializer(fieldSerializers);
        case UNION:
            UnionObjectInspector uoi = (UnionObjectInspector) oi;
            List<ObjectInspector> unionFields = uoi.getObjectInspectors();
            LBSerializer[] unionFieldSerializers = new LBSerializer[unionFields.size()];
            for (int idx = 0; idx < unionFieldSerializers.length; ++idx) {
                unionFieldSerializers[idx] = createLBSerializer(unionFields.get(idx));
            }
            return new LBUnionSerializer(unionFieldSerializers);
        default:
            throw new IllegalArgumentException("Unsupported category " + oi.getCategory());
    }
}

Example 65 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.

the class ObjectInspectorConverters method getConvertedOI.

/**
 * Utility function to convert from one object inspector type to another.
 * The output object inspector type should have all fields as settableOI type.
 * The above condition can be violated only if equalsCheck is true and inputOI is
 * equal to outputOI.
 * @param inputOI : input object inspector
 * @param outputOI : output object inspector
 * @param oiSettableProperties : The object inspector to isSettable mapping used to cache
 *                               intermediate results.
 * @param equalsCheck : Do we need to check if the inputOI and outputOI are the same?
 *                      true : If they are the same, we return the object inspector directly.
 *                      false : Do not perform an equality check on inputOI and outputOI
 * @return : The output object inspector containing all settable fields. The return value
 *           can contain non-settable fields only if inputOI equals outputOI and equalsCheck is
 *           true.
 */
public static ObjectInspector getConvertedOI(ObjectInspector inputOI, ObjectInspector outputOI, Map<ObjectInspector, Boolean> oiSettableProperties, boolean equalsCheck) {
    // 2. If the outputOI has all fields settable, return it
    if ((equalsCheck && inputOI.equals(outputOI)) || ObjectInspectorUtils.hasAllFieldsSettable(outputOI, oiSettableProperties) == true) {
        return outputOI;
    }
    // T is settable recursively i.e all the nested fields are also settable.
    switch(outputOI.getCategory()) {
        case PRIMITIVE:
            // Create a writable object inspector for primitive type and return it.
            PrimitiveObjectInspector primOutputOI = (PrimitiveObjectInspector) outputOI;
            return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primOutputOI.getTypeInfo());
        case STRUCT:
            StructObjectInspector structOutputOI = (StructObjectInspector) outputOI;
            // create a standard settable struct object inspector.
            List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
            List<String> structFieldNames = new ArrayList<String>(listFields.size());
            List<ObjectInspector> structFieldObjectInspectors = new ArrayList<ObjectInspector>(listFields.size());
            for (StructField listField : listFields) {
                structFieldNames.add(listField.getFieldName());
                // We need to make sure that the underlying fields are settable as well.
                // Hence, the recursive call for each field.
                // Note that equalsCheck is false while invoking getConvertedOI() because
                // we need to bypass the initial inputOI.equals(outputOI) check.
                structFieldObjectInspectors.add(getConvertedOI(listField.getFieldObjectInspector(), listField.getFieldObjectInspector(), oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardStructObjectInspector(structFieldNames, structFieldObjectInspectors);
        case LIST:
            ListObjectInspector listOutputOI = (ListObjectInspector) outputOI;
            // We need to make sure that the list element type is settable.
            return ObjectInspectorFactory.getStandardListObjectInspector(getConvertedOI(listOutputOI.getListElementObjectInspector(), listOutputOI.getListElementObjectInspector(), oiSettableProperties, false));
        case MAP:
            MapObjectInspector mapOutputOI = (MapObjectInspector) outputOI;
            // We need to make sure that the key type and the value types are settable.
            return ObjectInspectorFactory.getStandardMapObjectInspector(getConvertedOI(mapOutputOI.getMapKeyObjectInspector(), mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties, false), getConvertedOI(mapOutputOI.getMapValueObjectInspector(), mapOutputOI.getMapValueObjectInspector(), oiSettableProperties, false));
        case UNION:
            UnionObjectInspector unionOutputOI = (UnionObjectInspector) outputOI;
            // create a standard settable union object inspector
            List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
            List<ObjectInspector> unionFieldObjectInspectors = new ArrayList<ObjectInspector>(unionListFields.size());
            for (ObjectInspector listField : unionListFields) {
                // We need to make sure that all the field associated with the union are settable.
                unionFieldObjectInspectors.add(getConvertedOI(listField, listField, oiSettableProperties, false));
            }
            return ObjectInspectorFactory.getStandardUnionObjectInspector(unionFieldObjectInspectors);
        default:
            // Unsupported in-memory structure.
            throw new RuntimeException("Hive internal error: conversion of " + inputOI.getTypeName() + " to " + outputOI.getTypeName() + " not supported yet.");
    }
}