Examples with Category org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category used on opensource projects

Example 1 with Category

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category in project hive by apache.

the class TestVectorSerDeRow method serializeRow.

private Output serializeRow(Object[] row, VectorRandomRowSource source, SerializeWrite serializeWrite) throws HiveException, IOException {
    Output output = new Output();
    serializeWrite.set(output);
    PrimitiveTypeInfo[] primitiveTypeInfos = source.primitiveTypeInfos();
    for (int i = 0; i < primitiveTypeInfos.length; i++) {
        Object object = row[i];
        PrimitiveCategory primitiveCategory = primitiveTypeInfos[i].getPrimitiveCategory();
        switch(primitiveCategory) {
            case BOOLEAN:
                {
                    BooleanWritable expectedWritable = (BooleanWritable) object;
                    boolean value = expectedWritable.get();
                    serializeWrite.writeBoolean(value);
                }
                break;
            case BYTE:
                {
                    ByteWritable expectedWritable = (ByteWritable) object;
                    byte value = expectedWritable.get();
                    serializeWrite.writeByte(value);
                }
                break;
            case SHORT:
                {
                    ShortWritable expectedWritable = (ShortWritable) object;
                    short value = expectedWritable.get();
                    serializeWrite.writeShort(value);
                }
                break;
            case INT:
                {
                    IntWritable expectedWritable = (IntWritable) object;
                    int value = expectedWritable.get();
                    serializeWrite.writeInt(value);
                }
                break;
            case LONG:
                {
                    LongWritable expectedWritable = (LongWritable) object;
                    long value = expectedWritable.get();
                    serializeWrite.writeLong(value);
                }
                break;
            case DATE:
                {
                    DateWritable expectedWritable = (DateWritable) object;
                    Date value = expectedWritable.get();
                    serializeWrite.writeDate(value);
                }
                break;
            case FLOAT:
                {
                    FloatWritable expectedWritable = (FloatWritable) object;
                    float value = expectedWritable.get();
                    serializeWrite.writeFloat(value);
                }
                break;
            case DOUBLE:
                {
                    DoubleWritable expectedWritable = (DoubleWritable) object;
                    double value = expectedWritable.get();
                    serializeWrite.writeDouble(value);
                }
                break;
            case STRING:
                {
                    Text text = (Text) object;
                    serializeWrite.writeString(text.getBytes(), 0, text.getLength());
                }
                break;
            case CHAR:
                {
                    HiveCharWritable expectedWritable = (HiveCharWritable) object;
                    HiveChar value = expectedWritable.getHiveChar();
                    serializeWrite.writeHiveChar(value);
                }
                break;
            case VARCHAR:
                {
                    HiveVarcharWritable expectedWritable = (HiveVarcharWritable) object;
                    HiveVarchar value = expectedWritable.getHiveVarchar();
                    serializeWrite.writeHiveVarchar(value);
                }
                break;
            case BINARY:
                {
                    BytesWritable expectedWritable = (BytesWritable) object;
                    byte[] bytes = expectedWritable.getBytes();
                    int length = expectedWritable.getLength();
                    serializeWrite.writeBinary(bytes, 0, length);
                }
                break;
            case TIMESTAMP:
                {
                    TimestampWritable expectedWritable = (TimestampWritable) object;
                    Timestamp value = expectedWritable.getTimestamp();
                    serializeWrite.writeTimestamp(value);
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    HiveIntervalYearMonthWritable expectedWritable = (HiveIntervalYearMonthWritable) object;
                    HiveIntervalYearMonth value = expectedWritable.getHiveIntervalYearMonth();
                    serializeWrite.writeHiveIntervalYearMonth(value);
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    HiveIntervalDayTimeWritable expectedWritable = (HiveIntervalDayTimeWritable) object;
                    HiveIntervalDayTime value = expectedWritable.getHiveIntervalDayTime();
                    serializeWrite.writeHiveIntervalDayTime(value);
                }
                break;
            case DECIMAL:
                {
                    HiveDecimalWritable expectedWritable = (HiveDecimalWritable) object;
                    HiveDecimal value = expectedWritable.getHiveDecimal();
                    serializeWrite.writeHiveDecimal(value, ((DecimalTypeInfo) primitiveTypeInfos[i]).scale());
                }
                break;
            default:
                throw new HiveException("Unexpected primitive category " + primitiveCategory);
        }
    }
    return output;
}

Example 2 with Category

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category in project hive by apache.

the class TestVectorSerDeRow method deserializeAndVerify.

void deserializeAndVerify(Output output, DeserializeRead deserializeRead, VectorRandomRowSource source, Object[] expectedRow) throws HiveException, IOException {
    deserializeRead.set(output.getData(), 0, output.getLength());
    PrimitiveCategory[] primitiveCategories = source.primitiveCategories();
    for (int i = 0; i < primitiveCategories.length; i++) {
        Object expected = expectedRow[i];
        PrimitiveCategory primitiveCategory = primitiveCategories[i];
        PrimitiveTypeInfo primitiveTypeInfo = source.primitiveTypeInfos()[i];
        if (!deserializeRead.readNextField()) {
            throw new HiveException("Unexpected NULL when reading primitiveCategory " + primitiveCategory + " expected (" + expected.getClass().getName() + ", " + expected.toString() + ") " + " deserializeRead " + deserializeRead.getClass().getName());
        }
        switch(primitiveCategory) {
            case BOOLEAN:
                {
                    Boolean value = deserializeRead.currentBoolean;
                    BooleanWritable expectedWritable = (BooleanWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Boolean field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BYTE:
                {
                    Byte value = deserializeRead.currentByte;
                    ByteWritable expectedWritable = (ByteWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                    }
                }
                break;
            case SHORT:
                {
                    Short value = deserializeRead.currentShort;
                    ShortWritable expectedWritable = (ShortWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Short field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INT:
                {
                    Integer value = deserializeRead.currentInt;
                    IntWritable expectedWritable = (IntWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Int field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case LONG:
                {
                    Long value = deserializeRead.currentLong;
                    LongWritable expectedWritable = (LongWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Long field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DATE:
                {
                    DateWritable value = deserializeRead.currentDateWritable;
                    DateWritable expectedWritable = (DateWritable) expected;
                    if (!value.equals(expectedWritable)) {
                        TestCase.fail("Date field mismatch (expected " + expected.toString() + " found " + value.toString() + ")");
                    }
                }
                break;
            case FLOAT:
                {
                    Float value = deserializeRead.currentFloat;
                    FloatWritable expectedWritable = (FloatWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Float field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DOUBLE:
                {
                    Double value = deserializeRead.currentDouble;
                    DoubleWritable expectedWritable = (DoubleWritable) expected;
                    if (!value.equals(expectedWritable.get())) {
                        TestCase.fail("Double field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case STRING:
            case CHAR:
            case VARCHAR:
            case BINARY:
                {
                    byte[] stringBytes = Arrays.copyOfRange(deserializeRead.currentBytes, deserializeRead.currentBytesStart, deserializeRead.currentBytesStart + deserializeRead.currentBytesLength);
                    Text text = new Text(stringBytes);
                    String string = text.toString();
                    switch(primitiveCategory) {
                        case STRING:
                            {
                                Text expectedWritable = (Text) expected;
                                if (!string.equals(expectedWritable.toString())) {
                                    TestCase.fail("String field mismatch (expected '" + expectedWritable.toString() + "' found '" + string + "')");
                                }
                            }
                            break;
                        case CHAR:
                            {
                                HiveChar hiveChar = new HiveChar(string, ((CharTypeInfo) primitiveTypeInfo).getLength());
                                HiveCharWritable expectedWritable = (HiveCharWritable) expected;
                                if (!hiveChar.equals(expectedWritable.getHiveChar())) {
                                    TestCase.fail("Char field mismatch (expected '" + expectedWritable.getHiveChar() + "' found '" + hiveChar + "')");
                                }
                            }
                            break;
                        case VARCHAR:
                            {
                                HiveVarchar hiveVarchar = new HiveVarchar(string, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
                                HiveVarcharWritable expectedWritable = (HiveVarcharWritable) expected;
                                if (!hiveVarchar.equals(expectedWritable.getHiveVarchar())) {
                                    TestCase.fail("Varchar field mismatch (expected '" + expectedWritable.getHiveVarchar() + "' found '" + hiveVarchar + "')");
                                }
                            }
                            break;
                        case BINARY:
                            {
                                BytesWritable expectedWritable = (BytesWritable) expected;
                                if (stringBytes.length != expectedWritable.getLength()) {
                                    TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + stringBytes + ")");
                                }
                                byte[] expectedBytes = expectedWritable.getBytes();
                                for (int b = 0; b < stringBytes.length; b++) {
                                    if (stringBytes[b] != expectedBytes[b]) {
                                        TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + stringBytes + ")");
                                    }
                                }
                            }
                            break;
                        default:
                            throw new HiveException("Unexpected primitive category " + primitiveCategory);
                    }
                }
                break;
            case DECIMAL:
                {
                    HiveDecimal value = deserializeRead.currentHiveDecimalWritable.getHiveDecimal();
                    if (value == null) {
                        TestCase.fail("Decimal field evaluated to NULL");
                    }
                    HiveDecimalWritable expectedWritable = (HiveDecimalWritable) expected;
                    if (!value.equals(expectedWritable.getHiveDecimal())) {
                        DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                        int precision = decimalTypeInfo.getPrecision();
                        int scale = decimalTypeInfo.getScale();
                        TestCase.fail("Decimal field mismatch (expected " + expectedWritable.getHiveDecimal() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                    }
                }
                break;
            case TIMESTAMP:
                {
                    Timestamp value = deserializeRead.currentTimestampWritable.getTimestamp();
                    TimestampWritable expectedWritable = (TimestampWritable) expected;
                    if (!value.equals(expectedWritable.getTimestamp())) {
                        TestCase.fail("Timestamp field mismatch (expected " + expectedWritable.getTimestamp() + " found " + value.toString() + ")");
                    }
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    HiveIntervalYearMonth value = deserializeRead.currentHiveIntervalYearMonthWritable.getHiveIntervalYearMonth();
                    HiveIntervalYearMonthWritable expectedWritable = (HiveIntervalYearMonthWritable) expected;
                    HiveIntervalYearMonth expectedValue = expectedWritable.getHiveIntervalYearMonth();
                    if (!value.equals(expectedValue)) {
                        TestCase.fail("HiveIntervalYearMonth field mismatch (expected " + expectedValue + " found " + value.toString() + ")");
                    }
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    HiveIntervalDayTime value = deserializeRead.currentHiveIntervalDayTimeWritable.getHiveIntervalDayTime();
                    HiveIntervalDayTimeWritable expectedWritable = (HiveIntervalDayTimeWritable) expected;
                    HiveIntervalDayTime expectedValue = expectedWritable.getHiveIntervalDayTime();
                    if (!value.equals(expectedValue)) {
                        TestCase.fail("HiveIntervalDayTime field mismatch (expected " + expectedValue + " found " + value.toString() + ")");
                    }
                }
                break;
            default:
                throw new HiveException("Unexpected primitive category " + primitiveCategory);
        }
    }
    TestCase.assertTrue(deserializeRead.isEndOfInputReached());
}

Example 3 with Category

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category in project hive by apache.

the class VectorRandomRowSource method getWritableObject.

public static Object getWritableObject(int column, Object object, List<ObjectInspector> primitiveObjectInspectorList, PrimitiveCategory[] primitiveCategories, PrimitiveTypeInfo[] primitiveTypeInfos) {
    ObjectInspector objectInspector = primitiveObjectInspectorList.get(column);
    PrimitiveCategory primitiveCategory = primitiveCategories[column];
    PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[column];
    switch(primitiveCategory) {
        case BOOLEAN:
            return ((WritableBooleanObjectInspector) objectInspector).create((boolean) object);
        case BYTE:
            return ((WritableByteObjectInspector) objectInspector).create((byte) object);
        case SHORT:
            return ((WritableShortObjectInspector) objectInspector).create((short) object);
        case INT:
            return ((WritableIntObjectInspector) objectInspector).create((int) object);
        case LONG:
            return ((WritableLongObjectInspector) objectInspector).create((long) object);
        case DATE:
            return ((WritableDateObjectInspector) objectInspector).create((Date) object);
        case FLOAT:
            return ((WritableFloatObjectInspector) objectInspector).create((float) object);
        case DOUBLE:
            return ((WritableDoubleObjectInspector) objectInspector).create((double) object);
        case STRING:
            return ((WritableStringObjectInspector) objectInspector).create((String) object);
        case CHAR:
            {
                WritableHiveCharObjectInspector writableCharObjectInspector = new WritableHiveCharObjectInspector((CharTypeInfo) primitiveTypeInfo);
                return writableCharObjectInspector.create((HiveChar) object);
            }
        case VARCHAR:
            {
                WritableHiveVarcharObjectInspector writableVarcharObjectInspector = new WritableHiveVarcharObjectInspector((VarcharTypeInfo) primitiveTypeInfo);
                return writableVarcharObjectInspector.create((HiveVarchar) object);
            }
        case BINARY:
            return PrimitiveObjectInspectorFactory.writableBinaryObjectInspector.create((byte[]) object);
        case TIMESTAMP:
            return ((WritableTimestampObjectInspector) objectInspector).create((Timestamp) object);
        case INTERVAL_YEAR_MONTH:
            return ((WritableHiveIntervalYearMonthObjectInspector) objectInspector).create((HiveIntervalYearMonth) object);
        case INTERVAL_DAY_TIME:
            return ((WritableHiveIntervalDayTimeObjectInspector) objectInspector).create((HiveIntervalDayTime) object);
        case DECIMAL:
            {
                WritableHiveDecimalObjectInspector writableDecimalObjectInspector = new WritableHiveDecimalObjectInspector((DecimalTypeInfo) primitiveTypeInfo);
                HiveDecimalWritable result = (HiveDecimalWritable) writableDecimalObjectInspector.create((HiveDecimal) object);
                return result;
            }
        default:
            throw new Error("Unknown primitive category " + primitiveCategory);
    }
}

Example 4 with Category

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category in project hive by apache.

the class VectorRandomRowSource method randomObject.

public static Object randomObject(int column, Random r, PrimitiveCategory[] primitiveCategories, PrimitiveTypeInfo[] primitiveTypeInfos, String[] alphabets, boolean addEscapables, String needsEscapeStr) {
    PrimitiveCategory primitiveCategory = primitiveCategories[column];
    PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[column];
    try {
        switch(primitiveCategory) {
            case BOOLEAN:
                return Boolean.valueOf(r.nextInt(1) == 1);
            case BYTE:
                return Byte.valueOf((byte) r.nextInt());
            case SHORT:
                return Short.valueOf((short) r.nextInt());
            case INT:
                return Integer.valueOf(r.nextInt());
            case LONG:
                return Long.valueOf(r.nextLong());
            case DATE:
                return RandomTypeUtil.getRandDate(r);
            case FLOAT:
                return Float.valueOf(r.nextFloat() * 10 - 5);
            case DOUBLE:
                return Double.valueOf(r.nextDouble() * 10 - 5);
            case STRING:
            case CHAR:
            case VARCHAR:
                {
                    String result;
                    if (alphabets != null && alphabets[column] != null) {
                        result = RandomTypeUtil.getRandString(r, alphabets[column], r.nextInt(10));
                    } else {
                        result = RandomTypeUtil.getRandString(r);
                    }
                    if (addEscapables && result.length() > 0) {
                        int escapeCount = 1 + r.nextInt(2);
                        for (int i = 0; i < escapeCount; i++) {
                            int index = r.nextInt(result.length());
                            String begin = result.substring(0, index);
                            String end = result.substring(index);
                            Character needsEscapeChar = needsEscapeStr.charAt(r.nextInt(needsEscapeStr.length()));
                            result = begin + needsEscapeChar + end;
                        }
                    }
                    switch(primitiveCategory) {
                        case STRING:
                            return result;
                        case CHAR:
                            return new HiveChar(result, ((CharTypeInfo) primitiveTypeInfo).getLength());
                        case VARCHAR:
                            return new HiveVarchar(result, ((VarcharTypeInfo) primitiveTypeInfo).getLength());
                        default:
                            throw new Error("Unknown primitive category " + primitiveCategory);
                    }
                }
            case BINARY:
                return getRandBinary(r, 1 + r.nextInt(100));
            case TIMESTAMP:
                return RandomTypeUtil.getRandTimestamp(r);
            case INTERVAL_YEAR_MONTH:
                return getRandIntervalYearMonth(r);
            case INTERVAL_DAY_TIME:
                return getRandIntervalDayTime(r);
            case DECIMAL:
                return getRandHiveDecimal(r, (DecimalTypeInfo) primitiveTypeInfo);
            default:
                throw new Error("Unknown primitive category " + primitiveCategory);
        }
    } catch (Exception e) {
        throw new RuntimeException("randomObject failed on column " + column + " type " + primitiveCategory, e);
    }
}

Example 5 with Category

use of org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category in project hive by apache.

the class ObjectInspectorUtils method compareTypes.

/**
   * Compares two types identified by the given object inspectors. This method
   * compares the types as follows:
   * <ol>
   * <li>If the given inspectors do not belong to same category, the result is
   * negative.</li>
   * <li>If the given inspectors are for <code>PRIMITIVE</code> type, the result
   * is the comparison of their type names.</li>
   * <li>If the given inspectors are for <code>LIST</code> type, then the result
   * is recursive call to compare the type of list elements.</li>
   * <li>If the given inspectors are <code>MAP</code> type, then the result is a
   * recursive call to compare the map key and value types.</li>
   * <li>If the given inspectors are <code>STRUCT</code> type, then the result
   * is negative if they do not have the same number of fields. If they do have
   * the same number of fields, the result is a recursive call to compare each
   * of the field types.</li>
   * <li>If none of the above, the result is negative.</li>
   * </ol>
   * @param o1
   * @param o2
   * @return true if the given object inspectors represent the same types.
   */
public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) {
    Category c1 = o1.getCategory();
    Category c2 = o2.getCategory();
    // Return false if categories are not equal
    if (!c1.equals(c2)) {
        return false;
    }
    // If both categories are primitive return the comparison of type names.
    if (c1.equals(Category.PRIMITIVE)) {
        return o1.getTypeName().equals(o2.getTypeName());
    }
    // If lists, recursively compare the list element types
    if (c1.equals(Category.LIST)) {
        ObjectInspector child1 = ((ListObjectInspector) o1).getListElementObjectInspector();
        ObjectInspector child2 = ((ListObjectInspector) o2).getListElementObjectInspector();
        return compareTypes(child1, child2);
    }
    // If maps, recursively compare the key and value types
    if (c1.equals(Category.MAP)) {
        MapObjectInspector mapOI1 = (MapObjectInspector) o1;
        MapObjectInspector mapOI2 = (MapObjectInspector) o2;
        ObjectInspector childKey1 = mapOI1.getMapKeyObjectInspector();
        ObjectInspector childKey2 = mapOI2.getMapKeyObjectInspector();
        if (compareTypes(childKey1, childKey2)) {
            ObjectInspector childVal1 = mapOI1.getMapValueObjectInspector();
            ObjectInspector childVal2 = mapOI2.getMapValueObjectInspector();
            if (compareTypes(childVal1, childVal2)) {
                return true;
            }
        }
        return false;
    }
    // If structs, recursively compare the fields
    if (c1.equals(Category.STRUCT)) {
        StructObjectInspector structOI1 = (StructObjectInspector) o1;
        StructObjectInspector structOI2 = (StructObjectInspector) o2;
        List<? extends StructField> childFieldsList1 = structOI1.getAllStructFieldRefs();
        List<? extends StructField> childFieldsList2 = structOI2.getAllStructFieldRefs();
        if (childFieldsList1 == null && childFieldsList2 == null) {
            return true;
        } else if (childFieldsList1 == null || childFieldsList2 == null) {
            return false;
        } else if (childFieldsList1.size() != childFieldsList2.size()) {
            return false;
        }
        Iterator<? extends StructField> it1 = childFieldsList1.iterator();
        Iterator<? extends StructField> it2 = childFieldsList2.iterator();
        while (it1.hasNext()) {
            StructField field1 = it1.next();
            StructField field2 = it2.next();
            if (!compareTypes(field1.getFieldObjectInspector(), field2.getFieldObjectInspector())) {
                return false;
            }
        }
        return true;
    }
    if (c1.equals(Category.UNION)) {
        UnionObjectInspector uoi1 = (UnionObjectInspector) o1;
        UnionObjectInspector uoi2 = (UnionObjectInspector) o2;
        List<ObjectInspector> ois1 = uoi1.getObjectInspectors();
        List<ObjectInspector> ois2 = uoi2.getObjectInspectors();
        if (ois1 == null && ois2 == null) {
            return true;
        } else if (ois1 == null || ois2 == null) {
            return false;
        } else if (ois1.size() != ois2.size()) {
            return false;
        }
        Iterator<? extends ObjectInspector> it1 = ois1.iterator();
        Iterator<? extends ObjectInspector> it2 = ois2.iterator();
        while (it1.hasNext()) {
            if (!compareTypes(it1.next(), it2.next())) {
                return false;
            }
        }
        return true;
    }
    // Unknown category
    throw new RuntimeException("Unknown category encountered: " + c1);
}