Examples with HiveVarchar org.apache.hadoop.hive.common.type.HiveVarchar used on opensource projects

Example 81 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class VerifyLazy method lazyCompare.

public static boolean lazyCompare(TypeInfo typeInfo, Object lazyObject, Object expectedObject) {
    if (expectedObject == null) {
        if (lazyObject != null) {
            throw new RuntimeException("Expected object is null but object is not null " + lazyObject.toString() + " typeInfo " + typeInfo.toString());
        }
        return true;
    } else if (lazyObject == null) {
        throw new RuntimeException("Expected object is not null \"" + expectedObject.toString() + "\" typeInfo " + typeInfo.toString() + " but object is null");
    }
    if (lazyObject instanceof Writable) {
        if (!lazyObject.equals(expectedObject)) {
            throw new RuntimeException("Expected object " + expectedObject.toString() + " and actual object " + lazyObject.toString() + " is not equal typeInfo " + typeInfo.toString());
        }
        return true;
    }
    if (lazyObject instanceof LazyPrimitive) {
        Object primitiveObject = ((LazyPrimitive) lazyObject).getObject();
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
        switch(primitiveTypeInfo.getPrimitiveCategory()) {
            case BOOLEAN:
                {
                    if (!(primitiveObject instanceof LazyBoolean)) {
                        throw new RuntimeException("Expected LazyBoolean");
                    }
                    boolean value = ((LazyBoolean) primitiveObject).getWritableObject().get();
                    boolean expected = ((BooleanWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Boolean field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BYTE:
                {
                    if (!(primitiveObject instanceof LazyByte)) {
                        throw new RuntimeException("Expected LazyByte");
                    }
                    byte value = ((LazyByte) primitiveObject).getWritableObject().get();
                    byte expected = ((ByteWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                    }
                }
                break;
            case SHORT:
                {
                    if (!(primitiveObject instanceof LazyShort)) {
                        throw new RuntimeException("Expected LazyShort");
                    }
                    short value = ((LazyShort) primitiveObject).getWritableObject().get();
                    short expected = ((ShortWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Short field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INT:
                {
                    if (!(primitiveObject instanceof LazyInteger)) {
                        throw new RuntimeException("Expected LazyInteger");
                    }
                    int value = ((LazyInteger) primitiveObject).getWritableObject().get();
                    int expected = ((IntWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Int field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case LONG:
                {
                    if (!(primitiveObject instanceof LazyLong)) {
                        throw new RuntimeException("Expected LazyLong");
                    }
                    long value = ((LazyLong) primitiveObject).getWritableObject().get();
                    long expected = ((LongWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Long field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case FLOAT:
                {
                    if (!(primitiveObject instanceof LazyFloat)) {
                        throw new RuntimeException("Expected LazyFloat");
                    }
                    float value = ((LazyFloat) primitiveObject).getWritableObject().get();
                    float expected = ((FloatWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Float field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DOUBLE:
                {
                    if (!(primitiveObject instanceof LazyDouble)) {
                        throw new RuntimeException("Expected LazyDouble");
                    }
                    double value = ((LazyDouble) primitiveObject).getWritableObject().get();
                    double expected = ((DoubleWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Double field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case STRING:
                {
                    if (!(primitiveObject instanceof LazyString)) {
                        throw new RuntimeException("Text expected writable not Text");
                    }
                    Text value = ((LazyString) primitiveObject).getWritableObject();
                    Text expected = ((Text) expectedObject);
                    if (!value.equals(expected)) {
                        throw new RuntimeException("String field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case CHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveChar)) {
                        throw new RuntimeException("Expected LazyHiveChar");
                    }
                    HiveChar value = ((LazyHiveChar) primitiveObject).getWritableObject().getHiveChar();
                    HiveChar expected = ((HiveCharWritable) expectedObject).getHiveChar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveChar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case VARCHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveVarchar)) {
                        throw new RuntimeException("Expected LazyHiveVarchar");
                    }
                    HiveVarchar value = ((LazyHiveVarchar) primitiveObject).getWritableObject().getHiveVarchar();
                    HiveVarchar expected = ((HiveVarcharWritable) expectedObject).getHiveVarchar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveVarchar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case DECIMAL:
                {
                    if (!(primitiveObject instanceof LazyHiveDecimal)) {
                        throw new RuntimeException("Expected LazyDecimal");
                    }
                    HiveDecimal value = ((LazyHiveDecimal) primitiveObject).getWritableObject().getHiveDecimal();
                    HiveDecimal expected = ((HiveDecimalWritable) expectedObject).getHiveDecimal();
                    if (!value.equals(expected)) {
                        DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                        int precision = decimalTypeInfo.getPrecision();
                        int scale = decimalTypeInfo.getScale();
                        throw new RuntimeException("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                    }
                }
                break;
            case DATE:
                {
                    if (!(primitiveObject instanceof LazyDate)) {
                        throw new RuntimeException("Expected LazyDate");
                    }
                    Date value = ((LazyDate) primitiveObject).getWritableObject().get();
                    Date expected = ((DateWritableV2) expectedObject).get();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Date field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case TIMESTAMP:
                {
                    if (!(primitiveObject instanceof LazyTimestamp)) {
                        throw new RuntimeException("TimestampWritableV2 expected writable not TimestampWritableV2");
                    }
                    Timestamp value = ((LazyTimestamp) primitiveObject).getWritableObject().getTimestamp();
                    Timestamp expected = ((TimestampWritableV2) expectedObject).getTimestamp();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Timestamp field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalYearMonth)) {
                        throw new RuntimeException("Expected LazyHiveIntervalYearMonth");
                    }
                    HiveIntervalYearMonth value = ((LazyHiveIntervalYearMonth) primitiveObject).getWritableObject().getHiveIntervalYearMonth();
                    HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) expectedObject).getHiveIntervalYearMonth();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalYearMonth field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalDayTime)) {
                        throw new RuntimeException("Expected writable LazyHiveIntervalDayTime");
                    }
                    HiveIntervalDayTime value = ((LazyHiveIntervalDayTime) primitiveObject).getWritableObject().getHiveIntervalDayTime();
                    HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) expectedObject).getHiveIntervalDayTime();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalDayTime field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BINARY:
                {
                    if (!(primitiveObject instanceof LazyBinary)) {
                        throw new RuntimeException("Expected LazyBinary");
                    }
                    BytesWritable bytesWritable = ((LazyBinary) primitiveObject).getWritableObject();
                    byte[] value = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                    BytesWritable bytesWritableExpected = (BytesWritable) expectedObject;
                    byte[] expected = Arrays.copyOfRange(bytesWritableExpected.getBytes(), 0, bytesWritableExpected.getLength());
                    if (value.length != expected.length) {
                        throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                    }
                    for (int b = 0; b < value.length; b++) {
                        if (value[b] != expected[b]) {
                            throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                        }
                    }
                }
                break;
            default:
                throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
        }
    } else if (lazyObject instanceof LazyArray) {
        LazyArray lazyArray = (LazyArray) lazyObject;
        List<Object> list = lazyArray.getList();
        List<Object> expectedList = (List<Object>) expectedObject;
        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")" + " elementTypeInfo " + listTypeInfo.getListElementTypeInfo().toString());
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (typeInfo instanceof ListTypeInfo) {
        List<Object> list;
        if (lazyObject instanceof LazyBinaryArray) {
            list = ((LazyBinaryArray) lazyObject).getList();
        } else {
            list = (List<Object>) lazyObject;
        }
        List<Object> expectedList = (List<Object>) expectedObject;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")");
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (lazyObject instanceof LazyMap) {
        LazyMap lazyMap = (LazyMap) lazyObject;
        Map<Object, Object> map = lazyMap.getMap();
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (typeInfo instanceof MapTypeInfo) {
        Map<Object, Object> map;
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        if (lazyObject instanceof LazyBinaryMap) {
            map = ((LazyBinaryMap) lazyObject).getMap();
        } else {
            map = (Map<Object, Object>) lazyObject;
        }
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (lazyObject instanceof LazyStruct) {
        LazyStruct lazyStruct = (LazyStruct) lazyObject;
        List<Object> fields = lazyStruct.getFieldsAsList();
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (typeInfo instanceof StructTypeInfo) {
        ArrayList<Object> fields;
        if (lazyObject instanceof LazyBinaryStruct) {
            fields = ((LazyBinaryStruct) lazyObject).getFieldsAsList();
        } else {
            fields = (ArrayList<Object>) lazyObject;
        }
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (lazyObject instanceof LazyUnion) {
        LazyUnion union = (LazyUnion) lazyObject;
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        return lazyCompareUnion(unionTypeInfo, union, expectedUnion);
    } else if (typeInfo instanceof UnionTypeInfo) {
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        if (lazyObject instanceof LazyBinaryUnion) {
            return lazyCompareUnion(unionTypeInfo, (LazyBinaryUnion) lazyObject, expectedUnion);
        } else {
            return lazyCompareUnion(unionTypeInfo, (UnionObject) lazyObject, expectedUnion);
        }
    } else {
        System.err.println("Not implemented " + typeInfo.getClass().getName());
    }
    return true;
}

Example 82 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestDruidSerDe method testDruidObjectSerializerwithNullTimestamp.

@Test
public void testDruidObjectSerializerwithNullTimestamp() throws Exception {
    // Create, initialize, and test the SerDe
    DruidSerDe serDe = new DruidSerDe();
    Configuration conf = new Configuration();
    Properties tbl;
    // Mixed source (all types)
    tbl = createPropertiesSource(COLUMN_NAMES, COLUMN_TYPES);
    serDe.initialize(conf, tbl, null);
    Object[] row = new Object[] { null, new Text("dim1_val"), new HiveCharWritable(new HiveChar("dim2_v", 6)), new HiveVarcharWritable(new HiveVarchar("dim3_val", 8)), new DoubleWritable(10669.3D), new FloatWritable(10669.45F), new LongWritable(1113939), new IntWritable(1112123), new ShortWritable((short) 12), new ByteWritable((byte) 0), null // granularity
    };
    expectedEx.expect(NullPointerException.class);
    expectedEx.expectMessage("Timestamp column cannot have null value");
    // should fail as timestamp is null
    serializeObject(tbl, serDe, row, DRUID_WRITABLE);
}

Example 83 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestJsonSerDe method getData.

public List<Pair<Properties, HCatRecord>> getData() throws UnsupportedEncodingException {
    List<Pair<Properties, HCatRecord>> data = new ArrayList<Pair<Properties, HCatRecord>>();
    List<Object> rlist = new ArrayList<Object>(13);
    rlist.add(Byte.valueOf("123"));
    rlist.add(Short.valueOf("456"));
    rlist.add(Integer.valueOf(789));
    rlist.add(Long.valueOf(1000L));
    rlist.add(Double.valueOf(5.3D));
    rlist.add(Float.valueOf(2.39F));
    rlist.add("hcat\nand\nhadoop");
    rlist.add(null);
    List<Object> innerStruct = new ArrayList<Object>(2);
    innerStruct.add("abc");
    innerStruct.add("def");
    rlist.add(innerStruct);
    List<Integer> innerList = new ArrayList<Integer>();
    innerList.add(314);
    innerList.add(007);
    rlist.add(innerList);
    Map<Short, String> map = new HashMap<Short, String>(3);
    map.put(Short.valueOf("2"), "hcat is cool");
    map.put(Short.valueOf("3"), "is it?");
    map.put(Short.valueOf("4"), "or is it not?");
    rlist.add(map);
    rlist.add(Boolean.TRUE);
    List<Object> c1 = new ArrayList<Object>();
    List<Object> c1_1 = new ArrayList<Object>();
    c1_1.add(Integer.valueOf(12));
    List<Object> i2 = new ArrayList<Object>();
    List<Integer> ii1 = new ArrayList<Integer>();
    ii1.add(Integer.valueOf(13));
    ii1.add(Integer.valueOf(14));
    i2.add(ii1);
    Map<String, List<?>> ii2 = new HashMap<String, List<?>>();
    List<Integer> iii1 = new ArrayList<Integer>();
    iii1.add(Integer.valueOf(15));
    ii2.put("phew", iii1);
    i2.add(ii2);
    c1_1.add(i2);
    c1.add(c1_1);
    rlist.add(c1);
    // prec 5, scale 2
    rlist.add(HiveDecimal.create(new BigDecimal("123.45")));
    rlist.add(new HiveChar("hive\nchar", 10));
    rlist.add(new HiveVarchar("hive\nvarchar", 20));
    rlist.add(Date.valueOf("2014-01-07"));
    rlist.add(Timestamp.ofEpochMilli(System.currentTimeMillis()));
    rlist.add("hive\nbinary".getBytes("UTF-8"));
    List<Object> nlist = new ArrayList<Object>(13);
    // tinyint
    nlist.add(null);
    // smallint
    nlist.add(null);
    // int
    nlist.add(null);
    // bigint
    nlist.add(null);
    // double
    nlist.add(null);
    // float
    nlist.add(null);
    // string
    nlist.add(null);
    // string
    nlist.add(null);
    // struct
    nlist.add(null);
    // array
    nlist.add(null);
    // map
    nlist.add(null);
    // bool
    nlist.add(null);
    // complex
    nlist.add(null);
    // decimal(5,2)
    nlist.add(null);
    // char(10)
    nlist.add(null);
    // varchar(20)
    nlist.add(null);
    // date
    nlist.add(null);
    // timestamp
    nlist.add(null);
    // binary
    nlist.add(null);
    String typeString = "tinyint,smallint,int,bigint,double,float,string,string," + "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean," + "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>," + "decimal(5,2),char(10),varchar(20),date,timestamp,binary";
    Properties props = new Properties();
    props.put(serdeConstants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1,bd,hc,hvc,dt,ts,bin");
    props.put(serdeConstants.LIST_COLUMN_TYPES, typeString);
    // props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
    // props.put(Constants.SERIALIZATION_FORMAT, "1");
    data.add(new Pair<Properties, HCatRecord>(props, new DefaultHCatRecord(rlist)));
    data.add(new Pair<Properties, HCatRecord>(props, new DefaultHCatRecord(nlist)));
    return data;
}

Example 84 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class WritableHiveVarcharObjectInspector method getPrimitiveWithParams.

private HiveVarchar getPrimitiveWithParams(HiveVarcharWritable val) {
    HiveVarchar hv = new HiveVarchar();
    hv.setValue(val.getHiveVarchar(), getMaxLength());
    return hv;
}

Example 85 with HiveVarchar

use of org.apache.hadoop.hive.common.type.HiveVarchar in project hive by apache.

the class TestObjectInspectorConverters method convertVarChar.

private void convertVarChar() {
    Converter varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
    assertEquals("VarCharConverter", new HiveVarchar("TRUE", -1), varcharConverter.convert(Boolean.valueOf(true)));
    assertEquals("VarCharConverter", new HiveVarchar("FALSE", -1), varcharConverter.convert(Boolean.valueOf(false)));
    varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
    assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("TRUE", -1)), varcharConverter.convert(Boolean.valueOf(true)));
    assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("FALSE", -1)), varcharConverter.convert(Boolean.valueOf(false)));
    varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
    assertEquals("VarCharConverter", new HiveVarchar("0", -1), varcharConverter.convert(Integer.valueOf(0)));
    assertEquals("VarCharConverter", new HiveVarchar("1", -1), varcharConverter.convert(Integer.valueOf(1)));
    varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
    assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("0", -1)), varcharConverter.convert(Integer.valueOf(0)));
    assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("1", -1)), varcharConverter.convert(Integer.valueOf(1)));
    varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaHiveVarcharObjectInspector);
    assertEquals("VarCharConverter", new HiveVarchar("hive", -1), varcharConverter.convert(String.valueOf("hive")));
    varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.writableHiveVarcharObjectInspector);
    assertEquals("VarCharConverter", new HiveVarcharWritable(new HiveVarchar("hive", -1)), varcharConverter.convert(String.valueOf("hive")));
    // Varchar
    PrimitiveTypeInfo varchar5TI = (PrimitiveTypeInfo) TypeInfoFactory.getPrimitiveTypeInfo("varchar(5)");
    PrimitiveTypeInfo varchar30TI = (PrimitiveTypeInfo) TypeInfoFactory.getPrimitiveTypeInfo("varchar(30)");
    PrimitiveObjectInspector varchar5OI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(varchar5TI);
    PrimitiveObjectInspector varchar30OI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(varchar30TI);
    // Value should be truncated to varchar length 5
    varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector, varchar5OI);
    assertEquals("VarcharConverter", "100.0", varcharConverter.convert(HiveDecimal.create("100.001")).toString());
    varcharConverter = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector, varchar30OI);
    assertEquals("VarcharConverter", "100.001000000000000000", varcharConverter.convert(HiveDecimal.create("100.001")).toString());
}