Examples with TypeInfo org.apache.hadoop.hive.serde2.typeinfo.TypeInfo used on opensource projects

Example 41 with TypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo in project hive by apache.

the class VectorRandomRowSource method getObjectInspector.

private ObjectInspector getObjectInspector(TypeInfo typeInfo) {
    final ObjectInspector objectInspector;
    switch(typeInfo.getCategory()) {
        case PRIMITIVE:
            {
                final PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) typeInfo;
                objectInspector = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveType);
            }
            break;
        case MAP:
            {
                final MapTypeInfo mapType = (MapTypeInfo) typeInfo;
                final MapObjectInspector mapInspector = ObjectInspectorFactory.getStandardMapObjectInspector(getObjectInspector(mapType.getMapKeyTypeInfo()), getObjectInspector(mapType.getMapValueTypeInfo()));
                objectInspector = mapInspector;
            }
            break;
        case LIST:
            {
                final ListTypeInfo listType = (ListTypeInfo) typeInfo;
                final ListObjectInspector listInspector = ObjectInspectorFactory.getStandardListObjectInspector(getObjectInspector(listType.getListElementTypeInfo()));
                objectInspector = listInspector;
            }
            break;
        case STRUCT:
            {
                final StructTypeInfo structType = (StructTypeInfo) typeInfo;
                final List<TypeInfo> fieldTypes = structType.getAllStructFieldTypeInfos();
                final List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
                for (TypeInfo fieldType : fieldTypes) {
                    fieldInspectors.add(getObjectInspector(fieldType));
                }
                final StructObjectInspector structInspector = ObjectInspectorFactory.getStandardStructObjectInspector(structType.getAllStructFieldNames(), fieldInspectors);
                objectInspector = structInspector;
            }
            break;
        case UNION:
            {
                final UnionTypeInfo unionType = (UnionTypeInfo) typeInfo;
                final List<TypeInfo> fieldTypes = unionType.getAllUnionObjectTypeInfos();
                final List<ObjectInspector> fieldInspectors = new ArrayList<ObjectInspector>();
                for (TypeInfo fieldType : fieldTypes) {
                    fieldInspectors.add(getObjectInspector(fieldType));
                }
                final UnionObjectInspector unionInspector = ObjectInspectorFactory.getStandardUnionObjectInspector(fieldInspectors);
                objectInspector = unionInspector;
            }
            break;
        default:
            throw new RuntimeException("Unexpected category " + typeInfo.getCategory());
    }
    Preconditions.checkState(objectInspector != null);
    return objectInspector;
}

Example 42 with TypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo in project hive by apache.

the class VectorRandomRowSource method chooseSchema.

private void chooseSchema(SupportedTypes supportedTypes, int maxComplexDepth) {
    HashSet hashSet = null;
    final boolean allTypes;
    final boolean onlyOne = (r.nextInt(100) == 7);
    if (onlyOne) {
        columnCount = 1;
        allTypes = false;
    } else {
        allTypes = r.nextBoolean();
        if (allTypes) {
            switch(supportedTypes) {
                case ALL:
                    columnCount = possibleHivePrimitiveTypeNames.length + possibleHiveComplexTypeNames.length;
                    break;
                case ALL_EXCEPT_MAP:
                    columnCount = possibleHivePrimitiveTypeNames.length + possibleHiveComplexTypeNames.length - 1;
                    break;
                case PRIMITIVES:
                    columnCount = possibleHivePrimitiveTypeNames.length;
                    break;
            }
            hashSet = new HashSet<Integer>();
        } else {
            columnCount = 1 + r.nextInt(20);
        }
    }
    typeNames = new ArrayList<String>(columnCount);
    categories = new Category[columnCount];
    typeInfos = new TypeInfo[columnCount];
    objectInspectorList = new ArrayList<ObjectInspector>(columnCount);
    primitiveCategories = new PrimitiveCategory[columnCount];
    primitiveTypeInfos = new PrimitiveTypeInfo[columnCount];
    primitiveObjectInspectorList = new ArrayList<ObjectInspector>(columnCount);
    List<String> columnNames = new ArrayList<String>(columnCount);
    for (int c = 0; c < columnCount; c++) {
        columnNames.add(String.format("col%d", c));
        final String typeName;
        if (onlyOne) {
            typeName = getRandomTypeName(supportedTypes);
        } else {
            int typeNum;
            if (allTypes) {
                int maxTypeNum = 0;
                switch(supportedTypes) {
                    case PRIMITIVES:
                        maxTypeNum = possibleHivePrimitiveTypeNames.length;
                        break;
                    case ALL_EXCEPT_MAP:
                        maxTypeNum = possibleHivePrimitiveTypeNames.length + possibleHiveComplexTypeNames.length - 1;
                        break;
                    case ALL:
                        maxTypeNum = possibleHivePrimitiveTypeNames.length + possibleHiveComplexTypeNames.length;
                        break;
                }
                while (true) {
                    typeNum = r.nextInt(maxTypeNum);
                    Integer typeNumInteger = new Integer(typeNum);
                    if (!hashSet.contains(typeNumInteger)) {
                        hashSet.add(typeNumInteger);
                        break;
                    }
                }
            } else {
                if (supportedTypes == SupportedTypes.PRIMITIVES || r.nextInt(10) != 0) {
                    typeNum = r.nextInt(possibleHivePrimitiveTypeNames.length);
                } else {
                    typeNum = possibleHivePrimitiveTypeNames.length + r.nextInt(possibleHiveComplexTypeNames.length);
                    if (supportedTypes == SupportedTypes.ALL_EXCEPT_MAP) {
                        typeNum--;
                    }
                }
            }
            if (typeNum < possibleHivePrimitiveTypeNames.length) {
                typeName = possibleHivePrimitiveTypeNames[typeNum];
            } else {
                typeName = possibleHiveComplexTypeNames[typeNum - possibleHivePrimitiveTypeNames.length];
            }
        }
        String decoratedTypeName = getDecoratedTypeName(typeName, supportedTypes, 0, maxComplexDepth);
        final TypeInfo typeInfo;
        try {
            typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(decoratedTypeName);
        } catch (Exception e) {
            throw new RuntimeException("Cannot convert type name " + decoratedTypeName + " to a type " + e);
        }
        typeInfos[c] = typeInfo;
        final Category category = typeInfo.getCategory();
        categories[c] = category;
        ObjectInspector objectInspector = getObjectInspector(typeInfo);
        switch(category) {
            case PRIMITIVE:
                {
                    final PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
                    objectInspector = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveTypeInfo);
                    primitiveTypeInfos[c] = primitiveTypeInfo;
                    PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory();
                    primitiveCategories[c] = primitiveCategory;
                    primitiveObjectInspectorList.add(objectInspector);
                }
                break;
            case LIST:
            case MAP:
            case STRUCT:
            case UNION:
                primitiveObjectInspectorList.add(null);
                break;
            default:
                throw new RuntimeException("Unexpected catagory " + category);
        }
        objectInspectorList.add(objectInspector);
        if (category == Category.PRIMITIVE) {
        }
        typeNames.add(decoratedTypeName);
    }
    rowStructObjectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, objectInspectorList);
    alphabets = new String[columnCount];
}

Example 43 with TypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo in project carbondata by apache.

the class CarbonHiveRecordReader method initialize.

private void initialize(InputSplit inputSplit, Configuration conf) throws IOException {
    // The input split can contain single HDFS block or multiple blocks, so firstly get all the
    // blocks and then set them in the query model.
    List<CarbonHiveInputSplit> splitList;
    if (inputSplit instanceof CarbonHiveInputSplit) {
        splitList = new ArrayList<>(1);
        splitList.add((CarbonHiveInputSplit) inputSplit);
    } else {
        throw new RuntimeException("unsupported input split type: " + inputSplit);
    }
    List<TableBlockInfo> tableBlockInfoList = CarbonHiveInputSplit.createBlocks(splitList);
    queryModel.setTableBlockInfos(tableBlockInfoList);
    readSupport.initialize(queryModel.getProjectionColumns(), queryModel.getTable());
    try {
        carbonIterator = new ChunkRowIterator(queryExecutor.execute(queryModel));
    } catch (QueryExecutionException e) {
        throw new IOException(e.getMessage(), e.getCause());
    }
    final TypeInfo rowTypeInfo;
    final List<String> columnNames;
    List<TypeInfo> columnTypes;
    // Get column names and sort order
    final String colIds = conf.get("hive.io.file.readcolumn.ids");
    final String columnTypeProperty = conf.get(serdeConstants.LIST_COLUMN_TYPES);
    if (columnTypeProperty.length() == 0) {
        columnTypes = new ArrayList<TypeInfo>();
    } else {
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    }
    if (valueObj == null) {
        valueObj = new ArrayWritable(Writable.class, new Writable[columnTypes.size()]);
    }
    if (!colIds.equals("")) {
        String[] arraySelectedColId = colIds.split(",");
        columnIds = new int[arraySelectedColId.length];
        int columnId = 0;
        for (int j = 0; j < arraySelectedColId.length; j++) {
            columnId = Integer.parseInt(arraySelectedColId[j]);
            columnIds[j] = columnId;
        }
    }
}

Example 44 with TypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo in project carbondata by apache.

the class CarbonHiveSerDe method initialize.

@Override
public void initialize(@Nullable Configuration configuration, Properties tbl) throws SerDeException {
    final TypeInfo rowTypeInfo;
    final List<String> columnNames;
    final List<TypeInfo> columnTypes;
    // Get column names and sort order
    assert configuration != null;
    final String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
    final String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
    if (columnNameProperty.length() == 0) {
        columnNames = new ArrayList<String>();
    } else {
        columnNames = Arrays.asList(columnNameProperty.split(","));
    }
    if (columnTypeProperty.length() == 0) {
        columnTypes = new ArrayList<TypeInfo>();
    } else {
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    }
    // Create row related objects
    rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    this.objInspector = new CarbonObjectInspector((StructTypeInfo) rowTypeInfo);
    // Stats part
    serializedSize = 0;
    deserializedSize = 0;
    status = LAST_OPERATION.UNKNOWN;
}

Example 45 with TypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.TypeInfo in project incubator-gobblin by apache.

the class HiveAvroORCQueryGenerator method escapeHiveType.

/**
 * Escape the Hive nested field names.
 * @param type Primitive or nested Hive type.
 * @return Escaped Hive nested field.
 */
public static String escapeHiveType(String type) {
    TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(type);
    // Primitve
    if (ObjectInspector.Category.PRIMITIVE.equals(typeInfo.getCategory())) {
        return type;
    } else // List
    if (ObjectInspector.Category.LIST.equals(typeInfo.getCategory())) {
        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
        return org.apache.hadoop.hive.serde.serdeConstants.LIST_TYPE_NAME + "<" + escapeHiveType(listTypeInfo.getListElementTypeInfo().getTypeName()) + ">";
    } else // Map
    if (ObjectInspector.Category.MAP.equals(typeInfo.getCategory())) {
        MapTypeInfo mapTypeInfo = (MapTypeInfo) typeInfo;
        return org.apache.hadoop.hive.serde.serdeConstants.MAP_TYPE_NAME + "<" + escapeHiveType(mapTypeInfo.getMapKeyTypeInfo().getTypeName()) + "," + escapeHiveType(mapTypeInfo.getMapValueTypeInfo().getTypeName()) + ">";
    } else // Struct
    if (ObjectInspector.Category.STRUCT.equals(typeInfo.getCategory())) {
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        List<String> allStructFieldNames = structTypeInfo.getAllStructFieldNames();
        List<TypeInfo> allStructFieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
        StringBuilder sb = new StringBuilder();
        sb.append(serdeConstants.STRUCT_TYPE_NAME + "<");
        for (int i = 0; i < allStructFieldNames.size(); i++) {
            if (i > 0) {
                sb.append(",");
            }
            sb.append("`");
            sb.append(allStructFieldNames.get(i));
            sb.append("`");
            sb.append(":");
            sb.append(escapeHiveType(allStructFieldTypeInfos.get(i).getTypeName()));
        }
        sb.append(">");
        return sb.toString();
    } else // Union
    if (ObjectInspector.Category.UNION.equals(typeInfo.getCategory())) {
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        List<TypeInfo> allUnionObjectTypeInfos = unionTypeInfo.getAllUnionObjectTypeInfos();
        StringBuilder sb = new StringBuilder();
        sb.append(serdeConstants.UNION_TYPE_NAME + "<");
        for (int i = 0; i < allUnionObjectTypeInfos.size(); i++) {
            if (i > 0) {
                sb.append(",");
            }
            sb.append(escapeHiveType(allUnionObjectTypeInfos.get(i).getTypeName()));
        }
        sb.append(">");
        return sb.toString();
    } else {
        throw new RuntimeException("Unknown type encountered: " + type);
    }
}