Examples with StructTypeInfo org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo used on opensource projects

Example 91 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class VerifyLazy method lazyCompare.

public static boolean lazyCompare(TypeInfo typeInfo, Object lazyObject, Object expectedObject) {
    if (expectedObject == null) {
        if (lazyObject != null) {
            throw new RuntimeException("Expected object is null but object is not null " + lazyObject.toString() + " typeInfo " + typeInfo.toString());
        }
        return true;
    } else if (lazyObject == null) {
        throw new RuntimeException("Expected object is not null \"" + expectedObject.toString() + "\" typeInfo " + typeInfo.toString() + " but object is null");
    }
    if (lazyObject instanceof Writable) {
        if (!lazyObject.equals(expectedObject)) {
            throw new RuntimeException("Expected object " + expectedObject.toString() + " and actual object " + lazyObject.toString() + " is not equal typeInfo " + typeInfo.toString());
        }
        return true;
    }
    if (lazyObject instanceof LazyPrimitive) {
        Object primitiveObject = ((LazyPrimitive) lazyObject).getObject();
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
        switch(primitiveTypeInfo.getPrimitiveCategory()) {
            case BOOLEAN:
                {
                    if (!(primitiveObject instanceof LazyBoolean)) {
                        throw new RuntimeException("Expected LazyBoolean");
                    }
                    boolean value = ((LazyBoolean) primitiveObject).getWritableObject().get();
                    boolean expected = ((BooleanWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Boolean field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BYTE:
                {
                    if (!(primitiveObject instanceof LazyByte)) {
                        throw new RuntimeException("Expected LazyByte");
                    }
                    byte value = ((LazyByte) primitiveObject).getWritableObject().get();
                    byte expected = ((ByteWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                    }
                }
                break;
            case SHORT:
                {
                    if (!(primitiveObject instanceof LazyShort)) {
                        throw new RuntimeException("Expected LazyShort");
                    }
                    short value = ((LazyShort) primitiveObject).getWritableObject().get();
                    short expected = ((ShortWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Short field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INT:
                {
                    if (!(primitiveObject instanceof LazyInteger)) {
                        throw new RuntimeException("Expected LazyInteger");
                    }
                    int value = ((LazyInteger) primitiveObject).getWritableObject().get();
                    int expected = ((IntWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Int field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case LONG:
                {
                    if (!(primitiveObject instanceof LazyLong)) {
                        throw new RuntimeException("Expected LazyLong");
                    }
                    long value = ((LazyLong) primitiveObject).getWritableObject().get();
                    long expected = ((LongWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Long field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case FLOAT:
                {
                    if (!(primitiveObject instanceof LazyFloat)) {
                        throw new RuntimeException("Expected LazyFloat");
                    }
                    float value = ((LazyFloat) primitiveObject).getWritableObject().get();
                    float expected = ((FloatWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Float field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DOUBLE:
                {
                    if (!(primitiveObject instanceof LazyDouble)) {
                        throw new RuntimeException("Expected LazyDouble");
                    }
                    double value = ((LazyDouble) primitiveObject).getWritableObject().get();
                    double expected = ((DoubleWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Double field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case STRING:
                {
                    if (!(primitiveObject instanceof LazyString)) {
                        throw new RuntimeException("Text expected writable not Text");
                    }
                    Text value = ((LazyString) primitiveObject).getWritableObject();
                    Text expected = ((Text) expectedObject);
                    if (!value.equals(expected)) {
                        throw new RuntimeException("String field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case CHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveChar)) {
                        throw new RuntimeException("Expected LazyHiveChar");
                    }
                    HiveChar value = ((LazyHiveChar) primitiveObject).getWritableObject().getHiveChar();
                    HiveChar expected = ((HiveCharWritable) expectedObject).getHiveChar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveChar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case VARCHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveVarchar)) {
                        throw new RuntimeException("Expected LazyHiveVarchar");
                    }
                    HiveVarchar value = ((LazyHiveVarchar) primitiveObject).getWritableObject().getHiveVarchar();
                    HiveVarchar expected = ((HiveVarcharWritable) expectedObject).getHiveVarchar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveVarchar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case DECIMAL:
                {
                    if (!(primitiveObject instanceof LazyHiveDecimal)) {
                        throw new RuntimeException("Expected LazyDecimal");
                    }
                    HiveDecimal value = ((LazyHiveDecimal) primitiveObject).getWritableObject().getHiveDecimal();
                    HiveDecimal expected = ((HiveDecimalWritable) expectedObject).getHiveDecimal();
                    if (!value.equals(expected)) {
                        DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                        int precision = decimalTypeInfo.getPrecision();
                        int scale = decimalTypeInfo.getScale();
                        throw new RuntimeException("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                    }
                }
                break;
            case DATE:
                {
                    if (!(primitiveObject instanceof LazyDate)) {
                        throw new RuntimeException("Expected LazyDate");
                    }
                    Date value = ((LazyDate) primitiveObject).getWritableObject().get();
                    Date expected = ((DateWritableV2) expectedObject).get();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Date field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case TIMESTAMP:
                {
                    if (!(primitiveObject instanceof LazyTimestamp)) {
                        throw new RuntimeException("TimestampWritableV2 expected writable not TimestampWritableV2");
                    }
                    Timestamp value = ((LazyTimestamp) primitiveObject).getWritableObject().getTimestamp();
                    Timestamp expected = ((TimestampWritableV2) expectedObject).getTimestamp();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Timestamp field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalYearMonth)) {
                        throw new RuntimeException("Expected LazyHiveIntervalYearMonth");
                    }
                    HiveIntervalYearMonth value = ((LazyHiveIntervalYearMonth) primitiveObject).getWritableObject().getHiveIntervalYearMonth();
                    HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) expectedObject).getHiveIntervalYearMonth();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalYearMonth field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalDayTime)) {
                        throw new RuntimeException("Expected writable LazyHiveIntervalDayTime");
                    }
                    HiveIntervalDayTime value = ((LazyHiveIntervalDayTime) primitiveObject).getWritableObject().getHiveIntervalDayTime();
                    HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) expectedObject).getHiveIntervalDayTime();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalDayTime field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BINARY:
                {
                    if (!(primitiveObject instanceof LazyBinary)) {
                        throw new RuntimeException("Expected LazyBinary");
                    }
                    BytesWritable bytesWritable = ((LazyBinary) primitiveObject).getWritableObject();
                    byte[] value = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                    BytesWritable bytesWritableExpected = (BytesWritable) expectedObject;
                    byte[] expected = Arrays.copyOfRange(bytesWritableExpected.getBytes(), 0, bytesWritableExpected.getLength());
                    if (value.length != expected.length) {
                        throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                    }
                    for (int b = 0; b < value.length; b++) {
                        if (value[b] != expected[b]) {
                            throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                        }
                    }
                }
                break;
            default:
                throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
        }
    } else if (lazyObject instanceof LazyArray) {
        LazyArray lazyArray = (LazyArray) lazyObject;
        List<Object> list = lazyArray.getList();
        List<Object> expectedList = (List<Object>) expectedObject;
        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")" + " elementTypeInfo " + listTypeInfo.getListElementTypeInfo().toString());
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (typeInfo instanceof ListTypeInfo) {
        List<Object> list;
        if (lazyObject instanceof LazyBinaryArray) {
            list = ((LazyBinaryArray) lazyObject).getList();
        } else {
            list = (List<Object>) lazyObject;
        }
        List<Object> expectedList = (List<Object>) expectedObject;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")");
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (lazyObject instanceof LazyMap) {
        LazyMap lazyMap = (LazyMap) lazyObject;
        Map<Object, Object> map = lazyMap.getMap();
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (typeInfo instanceof MapTypeInfo) {
        Map<Object, Object> map;
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        if (lazyObject instanceof LazyBinaryMap) {
            map = ((LazyBinaryMap) lazyObject).getMap();
        } else {
            map = (Map<Object, Object>) lazyObject;
        }
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (lazyObject instanceof LazyStruct) {
        LazyStruct lazyStruct = (LazyStruct) lazyObject;
        List<Object> fields = lazyStruct.getFieldsAsList();
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (typeInfo instanceof StructTypeInfo) {
        ArrayList<Object> fields;
        if (lazyObject instanceof LazyBinaryStruct) {
            fields = ((LazyBinaryStruct) lazyObject).getFieldsAsList();
        } else {
            fields = (ArrayList<Object>) lazyObject;
        }
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (lazyObject instanceof LazyUnion) {
        LazyUnion union = (LazyUnion) lazyObject;
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        return lazyCompareUnion(unionTypeInfo, union, expectedUnion);
    } else if (typeInfo instanceof UnionTypeInfo) {
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        if (lazyObject instanceof LazyBinaryUnion) {
            return lazyCompareUnion(unionTypeInfo, (LazyBinaryUnion) lazyObject, expectedUnion);
        } else {
            return lazyCompareUnion(unionTypeInfo, (UnionObject) lazyObject, expectedUnion);
        }
    } else {
        System.err.println("Not implemented " + typeInfo.getClass().getName());
    }
    return true;
}

Example 92 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class LazyFactory method createLazyObjectInspector.

/**
 * Create a hierarchical ObjectInspector for LazyObject with the given typeInfo.
 *
 * @param typeInfo The type information for the LazyObject
 * @param separatorIndex The current level (for separators). List(array), struct uses 1 level of
 *          separator, and map uses 2 levels: the first one for delimiting entries, the second one
 *          for delimiting key and values.
 * @param lazyParams Params for lazy types
 * @param option the {@link ObjectInspectorOptions}
 * @return The ObjectInspector
 * @throws SerDeException
 */
public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, int separatorIndex, LazyObjectInspectorParameters lazyParams, ObjectInspectorOptions option) throws SerDeException {
    ObjectInspector.Category c = typeInfo.getCategory();
    switch(c) {
        case PRIMITIVE:
            return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector((PrimitiveTypeInfo) typeInfo, lazyParams);
        case MAP:
            return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(createLazyObjectInspector(((MapTypeInfo) typeInfo).getMapKeyTypeInfo(), separatorIndex + 2, lazyParams, option), createLazyObjectInspector(((MapTypeInfo) typeInfo).getMapValueTypeInfo(), separatorIndex + 2, lazyParams, option), LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex + 1), lazyParams);
        case LIST:
            return LazyObjectInspectorFactory.getLazySimpleListObjectInspector(createLazyObjectInspector(((ListTypeInfo) typeInfo).getListElementTypeInfo(), separatorIndex + 1, lazyParams, option), LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), lazyParams);
        case STRUCT:
            StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
            List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
            List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
            List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
            for (int i = 0; i < fieldTypeInfos.size(); i++) {
                fieldObjectInspectors.add(createLazyObjectInspector(fieldTypeInfos.get(i), separatorIndex + 1, lazyParams, option));
            }
            return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(fieldNames, fieldObjectInspectors, null, LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), lazyParams, option);
        case UNION:
            UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
            List<ObjectInspector> lazyOIs = new ArrayList<ObjectInspector>();
            for (TypeInfo uti : unionTypeInfo.getAllUnionObjectTypeInfos()) {
                lazyOIs.add(createLazyObjectInspector(uti, separatorIndex + 1, lazyParams, option));
            }
            return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs, LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex), lazyParams);
    }
    throw new RuntimeException("Hive LazySerDe Internal error.");
}

Example 93 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class SampleHBasePredicateDecomposer method getScanRange.

@Override
public HBaseScanRange getScanRange(List<IndexSearchCondition> searchConditions) throws Exception {
    Map<String, List<IndexSearchCondition>> fieldConds = new HashMap<String, List<IndexSearchCondition>>();
    for (IndexSearchCondition condition : searchConditions) {
        String fieldName = condition.getFields()[0];
        List<IndexSearchCondition> fieldCond = fieldConds.get(fieldName);
        if (fieldCond == null) {
            fieldConds.put(fieldName, fieldCond = new ArrayList<IndexSearchCondition>());
        }
        fieldCond.add(condition);
    }
    List<Filter> filters = new ArrayList<Filter>();
    HBaseScanRange range = new HBaseScanRange();
    StructTypeInfo type = (StructTypeInfo) keyMapping.columnType;
    for (String name : type.getAllStructFieldNames()) {
        List<IndexSearchCondition> fieldCond = fieldConds.get(name);
        if (fieldCond == null || fieldCond.size() > 2) {
            continue;
        }
        for (IndexSearchCondition condition : fieldCond) {
            if (condition.getConstantDesc().getValue() == null) {
                continue;
            }
            String comparisonOp = condition.getComparisonOp();
            String constantVal = String.valueOf(condition.getConstantDesc().getValue());
            byte[] valueAsBytes = toBinary(constantVal, FIXED_LENGTH, false, false);
            if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) {
                filters.add(new RowFilter(CompareOp.GREATER_OR_EQUAL, new BinaryComparator(valueAsBytes)));
            } else if (comparisonOp.endsWith("UDFOPGreaterThan")) {
                filters.add(new RowFilter(CompareOp.GREATER, new BinaryComparator(valueAsBytes)));
            } else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) {
                filters.add(new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator(valueAsBytes)));
            } else if (comparisonOp.endsWith("UDFOPLessThan")) {
                filters.add(new RowFilter(CompareOp.LESS, new BinaryComparator(valueAsBytes)));
            } else {
                throw new IOException(comparisonOp + " is not a supported comparison operator");
            }
        }
    }
    if (!filters.isEmpty()) {
        range.addFilter(new FilterList(Operator.MUST_PASS_ALL, filters));
    }
    return range;
}

Example 94 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class ColumnarStorageBench method createStructObjectInspector.

private StructObjectInspector createStructObjectInspector(Configuration conf) {
    // Create row related objects
    String columnNames = conf.get(IOConstants.COLUMNS);
    List<String> columnNamesList = DataWritableReadSupport.getColumnNames(columnNames);
    String columnTypes = conf.get(IOConstants.COLUMNS_TYPES);
    List<TypeInfo> columnTypesList = DataWritableReadSupport.getColumnTypes(columnTypes);
    TypeInfo rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNamesList, columnTypesList);
    return new ArrayWritableObjectInspector((StructTypeInfo) rowTypeInfo);
}

Example 95 with StructTypeInfo

use of org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo in project hive by apache.

the class HCatRecordObjectInspectorFactory method getHCatRecordObjectInspector.

/**
 * Returns HCatRecordObjectInspector given a StructTypeInfo type definition for the record to look into
 * @param typeInfo Type definition for the record to look into
 * @return appropriate HCatRecordObjectInspector
 * @throws SerDeException
 */
public static HCatRecordObjectInspector getHCatRecordObjectInspector(StructTypeInfo typeInfo) throws SerDeException {
    HCatRecordObjectInspector oi = cachedHCatRecordObjectInspectors.getIfPresent(typeInfo);
    if (oi == null) {
        LOG.debug("Got asked for OI for {} [{} ]", typeInfo.getCategory(), typeInfo.getTypeName());
        switch(typeInfo.getCategory()) {
            case STRUCT:
                StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
                List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
                List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
                List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
                for (int i = 0; i < fieldTypeInfos.size(); i++) {
                    fieldObjectInspectors.add(getStandardObjectInspectorFromTypeInfo(fieldTypeInfos.get(i)));
                }
                oi = new HCatRecordObjectInspector(fieldNames, fieldObjectInspectors);
                break;
            default:
                // - anything else is an error. Return null as the inspector.
                throw new SerDeException("TypeInfo [" + typeInfo.getTypeName() + "] was not of struct type - HCatRecord expected struct type, got [" + typeInfo.getCategory().toString() + "]");
        }
        cachedHCatRecordObjectInspectors.put(typeInfo, oi);
    }
    return oi;
}