Examples with StructTypeInfo org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo used on opensource projects

Example 36 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class TestVectorStructField method doOneStructFieldTest.

private void doOneStructFieldTest(Random random, StructTypeInfo structTypeInfo, String structTypeName, int fieldIndex) throws Exception {
    List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>();
    List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList = new ArrayList<DataTypePhysicalVariation>();
    List<String> columns = new ArrayList<String>();
    int columnNum = 1;
    generationSpecList.add(GenerationSpec.createSameType(structTypeInfo));
    explicitDataTypePhysicalVariationList.add(DataTypePhysicalVariation.NONE);
    ExprNodeDesc col1Expr;
    String columnName = "col" + (columnNum++);
    col1Expr = new ExprNodeColumnDesc(structTypeInfo, columnName, "table", false);
    columns.add(columnName);
    ObjectInspector structObjectInspector = VectorRandomRowSource.getObjectInspector(structTypeInfo);
    List<ObjectInspector> objectInspectorList = new ArrayList<ObjectInspector>();
    objectInspectorList.add(structObjectInspector);
    List<ExprNodeDesc> children = new ArrayList<ExprNodeDesc>();
    children.add(col1Expr);
    // ----------------------------------------------------------------------------------------------
    String[] columnNames = columns.toArray(new String[0]);
    VectorRandomRowSource rowSource = new VectorRandomRowSource();
    rowSource.initGenerationSpecSchema(random, generationSpecList, /* maxComplexDepth */
    0, /* allowNull */
    true, /* isUnicodeOk */
    true, explicitDataTypePhysicalVariationList);
    Object[][] randomRows = rowSource.randomRows(100000);
    VectorRandomBatchSource batchSource = VectorRandomBatchSource.createInterestingBatches(random, rowSource, randomRows, null);
    List<String> fieldNameList = structTypeInfo.getAllStructFieldNames();
    List<TypeInfo> fieldTypeInfoList = structTypeInfo.getAllStructFieldTypeInfos();
    String randomFieldName = fieldNameList.get(fieldIndex);
    TypeInfo outputTypeInfo = fieldTypeInfoList.get(fieldIndex);
    ExprNodeFieldDesc exprNodeFieldDesc = new ExprNodeFieldDesc(outputTypeInfo, col1Expr, randomFieldName, /* isList */
    false);
    final int rowCount = randomRows.length;
    Object[][] resultObjectsArray = new Object[StructFieldTestMode.count][];
    for (int i = 0; i < StructFieldTestMode.count; i++) {
        Object[] resultObjects = new Object[rowCount];
        resultObjectsArray[i] = resultObjects;
        StructFieldTestMode negativeTestMode = StructFieldTestMode.values()[i];
        switch(negativeTestMode) {
            case ROW_MODE:
                doRowStructFieldTest(structTypeInfo, columns, children, exprNodeFieldDesc, randomRows, rowSource.rowStructObjectInspector(), outputTypeInfo, resultObjects);
                break;
            case VECTOR_EXPRESSION:
                doVectorStructFieldTest(structTypeInfo, columns, columnNames, rowSource.typeInfos(), rowSource.dataTypePhysicalVariations(), children, exprNodeFieldDesc, negativeTestMode, batchSource, exprNodeFieldDesc.getWritableObjectInspector(), outputTypeInfo, resultObjects);
                break;
            default:
                throw new RuntimeException("Unexpected Negative operator test mode " + negativeTestMode);
        }
    }
    for (int i = 0; i < rowCount; i++) {
        // Row-mode is the expected value.
        Object expectedResult = resultObjectsArray[0][i];
        for (int v = 1; v < StructFieldTestMode.count; v++) {
            Object vectorResult = resultObjectsArray[v][i];
            if (expectedResult == null || vectorResult == null) {
                if (expectedResult != null || vectorResult != null) {
                    Assert.fail("Row " + i + " structTypeName " + structTypeName + " outputTypeName " + outputTypeInfo.getTypeName() + " " + StructFieldTestMode.values()[v] + " result is NULL " + (vectorResult == null) + " does not match row-mode expected result is NULL " + (expectedResult == null) + " row values " + Arrays.toString(randomRows[i]));
                }
            } else {
                if (!expectedResult.equals(vectorResult)) {
                    Assert.fail("Row " + i + " structTypeName " + structTypeName + " outputTypeName " + outputTypeInfo.getTypeName() + " " + StructFieldTestMode.values()[v] + " result " + vectorResult.toString() + " (" + vectorResult.getClass().getSimpleName() + ")" + " does not match row-mode expected result " + expectedResult.toString() + " (" + expectedResult.getClass().getSimpleName() + ")" + " row values " + Arrays.toString(randomRows[i]));
                }
            }
        }
    }
}

Example 37 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class BatchToRowReader method nextStruct.

public StructType nextStruct(ColumnVector vector, int row, StructTypeInfo schema, Object previous) {
    if (vector.isRepeating) {
        row = 0;
    }
    if (vector.noNulls || !vector.isNull[row]) {
        List<TypeInfo> childrenTypes = schema.getAllStructFieldTypeInfos();
        StructType result = createStructObject(previous, childrenTypes);
        StructColumnVector struct = (StructColumnVector) vector;
        for (int f = 0; f < childrenTypes.size(); ++f) {
            setStructCol(result, f, nextValue(struct.fields[f], row, childrenTypes.get(f), getStructCol(result, f)));
        }
        return result;
    } else {
        return null;
    }
}

Example 38 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class PTFDeserializer method getTypeMap.

private static List<? extends Object>[] getTypeMap(StructObjectInspector oi) {
    StructTypeInfo t = (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(oi);
    List<String> fnames = t.getAllStructFieldNames();
    List<TypeInfo> fields = t.getAllStructFieldTypeInfos();
    return new List<?>[] { fnames, fields };
}

Example 39 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class TestDataWritableWriter method getObjectInspector.

private ArrayWritableObjectInspector getObjectInspector(final String columnNames, final String columnTypes) {
    List<TypeInfo> columnTypeList = createHiveTypeInfoFrom(columnTypes);
    List<String> columnNameList = createHiveColumnsFrom(columnNames);
    StructTypeInfo rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNameList, columnTypeList);
    return new ArrayWritableObjectInspector(rowTypeInfo);
}

Example 40 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class TestParquetRowGroupFilter method getObjectInspector.

private ArrayWritableObjectInspector getObjectInspector(final String columnNames, final String columnTypes) {
    List<TypeInfo> columnTypeList = createHiveTypeInfoFrom(columnTypes);
    List<String> columnNameList = createHiveColumnsFrom(columnNames);
    StructTypeInfo rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNameList, columnTypeList);
    return new ArrayWritableObjectInspector(rowTypeInfo);
}