Search in sources :

Example 1 with StandardConstantMapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardConstantMapObjectInspector in project hive by apache.

the class StatsUtils method getSizeOfComplexTypes.

/**
   * Get the size of complex data types
   * @param conf
   *          - hive conf
   * @param oi
   *          - object inspector
   * @return raw data size
   */
public static long getSizeOfComplexTypes(HiveConf conf, ObjectInspector oi) {
    long result = 0;
    int length = 0;
    int listEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_LIST_NUM_ENTRIES);
    int mapEntries = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_STATS_MAP_NUM_ENTRIES);
    switch(oi.getCategory()) {
        case PRIMITIVE:
            String colTypeLowerCase = oi.getTypeName().toLowerCase();
            if (colTypeLowerCase.equals(serdeConstants.STRING_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME)) {
                int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
                result += JavaDataModel.get().lengthForStringOfLength(avgColLen);
            } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) {
                int avgColLen = (int) getAvgColLenOf(conf, oi, colTypeLowerCase);
                result += JavaDataModel.get().lengthForByteArrayOfSize(avgColLen);
            } else {
                result += getAvgColLenOfFixedLengthTypes(colTypeLowerCase);
            }
            break;
        case LIST:
            if (oi instanceof StandardConstantListObjectInspector) {
                // constant list projection of known length
                StandardConstantListObjectInspector scloi = (StandardConstantListObjectInspector) oi;
                length = scloi.getWritableConstantValue().size();
                // check if list elements are primitive or Objects
                ObjectInspector leoi = scloi.getListElementObjectInspector();
                if (leoi.getCategory().equals(ObjectInspector.Category.PRIMITIVE)) {
                    result += getSizeOfPrimitiveTypeArraysFromType(leoi.getTypeName(), length);
                } else {
                    result += JavaDataModel.get().lengthForObjectArrayOfSize(length);
                }
            } else {
                StandardListObjectInspector sloi = (StandardListObjectInspector) oi;
                // list overhead + (configured number of element in list * size of element)
                long elemSize = getSizeOfComplexTypes(conf, sloi.getListElementObjectInspector());
                result += JavaDataModel.get().arrayList() + (listEntries * elemSize);
            }
            break;
        case MAP:
            if (oi instanceof StandardConstantMapObjectInspector) {
                // constant map projection of known length
                StandardConstantMapObjectInspector scmoi = (StandardConstantMapObjectInspector) oi;
                result += getSizeOfMap(scmoi);
            } else {
                StandardMapObjectInspector smoi = (StandardMapObjectInspector) oi;
                result += getSizeOfComplexTypes(conf, smoi.getMapKeyObjectInspector());
                result += getSizeOfComplexTypes(conf, smoi.getMapValueObjectInspector());
                // hash map overhead
                result += JavaDataModel.get().hashMap(mapEntries);
            }
            break;
        case STRUCT:
            if (oi instanceof StandardConstantStructObjectInspector) {
                // constant map projection of known length
                StandardConstantStructObjectInspector scsoi = (StandardConstantStructObjectInspector) oi;
                result += getSizeOfStruct(scsoi);
            } else {
                StructObjectInspector soi = (StructObjectInspector) oi;
                // add constant object overhead for struct
                result += JavaDataModel.get().object();
                // add constant struct field names references overhead
                result += soi.getAllStructFieldRefs().size() * JavaDataModel.get().ref();
                for (StructField field : soi.getAllStructFieldRefs()) {
                    result += getSizeOfComplexTypes(conf, field.getFieldObjectInspector());
                }
            }
            break;
        case UNION:
            UnionObjectInspector uoi = (UnionObjectInspector) oi;
            // add constant object overhead for union
            result += JavaDataModel.get().object();
            // add constant size for unions tags
            result += uoi.getObjectInspectors().size() * JavaDataModel.get().primitive1();
            for (ObjectInspector foi : uoi.getObjectInspectors()) {
                result += getSizeOfComplexTypes(conf, foi);
            }
            break;
        default:
            break;
    }
    return result;
}
Also used : WritableIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector) WritableByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) WritableDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) StandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) StringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector) StandardConstantListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantListObjectInspector) StandardConstantMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantMapObjectInspector) WritableStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector) HiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector) HiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector) WritableBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector) WritableBinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector) WritableTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector) StandardConstantStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector) WritableShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector) StandardMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector) BinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector) WritableFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector) WritableLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector) WritableDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector) ConstantObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector) WritableHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector) StandardConstantMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantMapObjectInspector) StandardMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector) StructField(org.apache.hadoop.hive.serde2.objectinspector.StructField) StandardConstantListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantListObjectInspector) StandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector) StandardConstantStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) StandardConstantStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector)

Example 2 with StandardConstantMapObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardConstantMapObjectInspector in project hive by apache.

the class StatsUtils method getSizeOfMap.

/**
   * Estimate the size of map object
   * @param scmoi
   *          - object inspector
   * @return size of map
   */
public static long getSizeOfMap(StandardConstantMapObjectInspector scmoi) {
    Map<?, ?> map = scmoi.getWritableConstantValue();
    ObjectInspector koi = scmoi.getMapKeyObjectInspector();
    ObjectInspector voi = scmoi.getMapValueObjectInspector();
    long result = 0;
    for (Map.Entry<?, ?> entry : map.entrySet()) {
        result += getWritableSize(koi, entry.getKey());
        result += getWritableSize(voi, entry.getValue());
    }
    // add additional overhead of each map entries
    result += JavaDataModel.get().hashMap(map.entrySet().size());
    return result;
}
Also used : WritableIntObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector) WritableByteObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector) UnionObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector) WritableDoubleObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) StandardListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) StringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector) StandardConstantListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantListObjectInspector) StandardConstantMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantMapObjectInspector) WritableStringObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector) HiveVarcharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector) HiveCharObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector) WritableBooleanObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector) WritableBinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector) WritableTimestampObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector) StandardConstantStructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector) WritableShortObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector) StandardMapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector) BinaryObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector) WritableFloatObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector) WritableLongObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector) WritableDateObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector) ConstantObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector) WritableHiveDecimalObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector) Map(java.util.Map)

Aggregations

ConstantObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector)2 ObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)2 StandardConstantListObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StandardConstantListObjectInspector)2 StandardConstantMapObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StandardConstantMapObjectInspector)2 StandardConstantStructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StandardConstantStructObjectInspector)2 StandardListObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector)2 StandardMapObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector)2 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)2 UnionObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector)2 BinaryObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector)2 HiveCharObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector)2 HiveVarcharObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector)2 StringObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector)2 WritableBinaryObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBinaryObjectInspector)2 WritableBooleanObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector)2 WritableByteObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector)2 WritableDateObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDateObjectInspector)2 WritableDoubleObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector)2 WritableFloatObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector)2 WritableHiveDecimalObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector)2