Examples with StructTypeInfo org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo used on opensource projects

Example 86 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class TestVectorStructField method doStructFieldTests.

private void doStructFieldTests(Random random) throws Exception {
    String structTypeName = VectorRandomRowSource.getDecoratedTypeName(random, "struct", SupportedTypes.ALL, /* allowedTypeNameSet */
    null, /* depth */
    0, /* maxDepth */
    2);
    StructTypeInfo structTypeInfo = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromTypeString(structTypeName);
    List<String> fieldNameList = structTypeInfo.getAllStructFieldNames();
    final int fieldCount = fieldNameList.size();
    for (int fieldIndex = 0; fieldIndex < fieldCount; fieldIndex++) {
        doOneStructFieldTest(random, structTypeInfo, structTypeName, fieldIndex);
    }
}

Example 87 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class ArrowColumnarBatchSerDe method toStructListTypeInfo.

static ListTypeInfo toStructListTypeInfo(MapTypeInfo mapTypeInfo) {
    final StructTypeInfo structTypeInfo = new StructTypeInfo();
    structTypeInfo.setAllStructFieldNames(Lists.newArrayList("key", "value"));
    structTypeInfo.setAllStructFieldTypeInfos(Lists.newArrayList(mapTypeInfo.getMapKeyTypeInfo(), mapTypeInfo.getMapValueTypeInfo()));
    final ListTypeInfo structListTypeInfo = new ListTypeInfo();
    structListTypeInfo.setListElementTypeInfo(structTypeInfo);
    return structListTypeInfo;
}

Example 88 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class Serializer method writeList.

private void writeList(ListVector arrowVector, ListColumnVector hiveVector, ListTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative, boolean isMapDataType) {
    final int OFFSET_WIDTH = 4;
    final TypeInfo elementTypeInfo = typeInfo.getListElementTypeInfo();
    final ColumnVector hiveElementVector = hiveVector == null ? null : hiveVector.child;
    // If the call is coming from writeMap(), then the List type should be non-nullable.
    FieldType elementFieldType = (isMapDataType) ? (new FieldType(false, toArrowType(elementTypeInfo), null)) : (toFieldType(elementTypeInfo));
    final FieldVector arrowElementVector = (FieldVector) arrowVector.addOrGetVector(elementFieldType).getVector();
    VectorizedRowBatch correctedVrb = vectorizedRowBatch;
    int correctedSize = hiveVector == null ? 0 : hiveVector.childCount;
    if (vectorizedRowBatch.selectedInUse) {
        correctedVrb = correctSelectedAndSize(vectorizedRowBatch, hiveVector);
        correctedSize = correctedVrb.size;
    }
    arrowElementVector.setInitialCapacity(correctedSize);
    arrowElementVector.allocateNew();
    // writeStruct() with the same flag value, as the map is converted as a list of structs.
    if (isMapDataType) {
        writeStruct((NonNullableStructVector) arrowElementVector, (StructColumnVector) hiveElementVector, (StructTypeInfo) elementTypeInfo, correctedSize, correctedVrb, isNative, isMapDataType);
    } else {
        write(arrowElementVector, hiveElementVector, elementTypeInfo, correctedSize, correctedVrb, isNative);
    }
    int nextOffset = 0;
    for (int rowIndex = 0; rowIndex < size; rowIndex++) {
        int selectedIndex = rowIndex;
        if (vectorizedRowBatch.selectedInUse) {
            selectedIndex = vectorizedRowBatch.selected[rowIndex];
        }
        if (hiveVector == null || hiveVector.isNull[selectedIndex]) {
            arrowVector.getOffsetBuffer().setInt(rowIndex * OFFSET_WIDTH, nextOffset);
        } else {
            arrowVector.getOffsetBuffer().setInt(rowIndex * OFFSET_WIDTH, nextOffset);
            nextOffset += (int) hiveVector.lengths[selectedIndex];
            arrowVector.setNotNull(rowIndex);
        }
    }
    arrowVector.getOffsetBuffer().setInt(size * OFFSET_WIDTH, nextOffset);
}

Example 89 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class Vectorizer method validateStructInExpression.

private boolean validateStructInExpression(ExprNodeDesc desc, String expressionTitle, VectorExpressionDescriptor.Mode mode) {
    for (ExprNodeDesc d : desc.getChildren()) {
        TypeInfo typeInfo = d.getTypeInfo();
        if (typeInfo.getCategory() != Category.STRUCT) {
            return false;
        }
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
        List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
        final int fieldCount = fieldTypeInfos.size();
        for (int f = 0; f < fieldCount; f++) {
            TypeInfo fieldTypeInfo = fieldTypeInfos.get(f);
            Category category = fieldTypeInfo.getCategory();
            if (category != Category.PRIMITIVE) {
                setExpressionIssue(expressionTitle, "Cannot vectorize struct field " + fieldNames.get(f) + " of type " + fieldTypeInfo.getTypeName());
                return false;
            }
            PrimitiveTypeInfo fieldPrimitiveTypeInfo = (PrimitiveTypeInfo) fieldTypeInfo;
            InConstantType inConstantType = VectorizationContext.getInConstantTypeFromPrimitiveCategory(fieldPrimitiveTypeInfo.getPrimitiveCategory());
            // For now, limit the data types we support for Vectorized Struct IN().
            if (inConstantType != InConstantType.INT_FAMILY && inConstantType != InConstantType.FLOAT_FAMILY && inConstantType != InConstantType.STRING_FAMILY) {
                setExpressionIssue(expressionTitle, "Cannot vectorize struct field " + fieldNames.get(f) + " of type " + fieldTypeInfo.getTypeName());
                return false;
            }
        }
    }
    return true;
}

Example 90 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class VerifyLazy method lazyCompareStruct.

public static boolean lazyCompareStruct(StructTypeInfo structTypeInfo, List<Object> fields, List<Object> expectedFields) {
    List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
    final int size = fieldTypeInfos.size();
    for (int i = 0; i < size; i++) {
        Object lazyEleObj = fields.get(i);
        Object expectedEleObj = expectedFields.get(i);
        if (!lazyCompare(fieldTypeInfos.get(i), lazyEleObj, expectedEleObj)) {
            throw new RuntimeException("SerDe deserialized value does not match");
        }
    }
    return true;
}