Examples with UnionObject org.apache.hadoop.hive.serde2.objectinspector.UnionObject used on opensource projects

Example 1 with UnionObject

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObject in project cdap by caskdata.

the class StandardObjectInspectorsTest method testStandardUnionObjectInspector.

@Test
public void testStandardUnionObjectInspector() throws Throwable {
    try {
        ArrayList<ObjectInspector> objectInspectors = new ArrayList<>();
        // add primitive types
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
        // add a list
        objectInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector));
        // add a map
        objectInspectors.add(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector));
        // add a struct
        List<String> fieldNames = new ArrayList<>();
        fieldNames.add("myDouble");
        fieldNames.add("myLong");
        ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<>();
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
        objectInspectors.add(ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors));
        StandardUnionObjectInspector uoi1 = ObjectInspectorFactory.getStandardUnionObjectInspector(objectInspectors);
        StandardUnionObjectInspector uoi2 = ObjectInspectorFactory.getStandardUnionObjectInspector((ArrayList<ObjectInspector>) objectInspectors.clone());
        Assert.assertEquals(uoi1, uoi2);
        Assert.assertEquals(ObjectInspectorUtils.getObjectInspectorName(uoi1), ObjectInspectorUtils.getObjectInspectorName(uoi2));
        Assert.assertTrue(ObjectInspectorUtils.compareTypes(uoi1, uoi2));
        // compareSupported returns false because Union can contain
        // an object of Map
        Assert.assertFalse(ObjectInspectorUtils.compareSupported(uoi1));
        // construct unionObjectInspector without Map field.
        ArrayList<ObjectInspector> ois = (ArrayList<ObjectInspector>) objectInspectors.clone();
        ois.set(4, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        Assert.assertTrue(ObjectInspectorUtils.compareSupported(ObjectInspectorFactory.getStandardUnionObjectInspector(ois)));
        // metadata
        Assert.assertEquals(Category.UNION, uoi1.getCategory());
        List<? extends ObjectInspector> uois = uoi1.getObjectInspectors();
        Assert.assertEquals(6, uois.size());
        for (int i = 0; i < 6; i++) {
            Assert.assertEquals(objectInspectors.get(i), uois.get(i));
        }
        StringBuilder unionTypeName = new StringBuilder();
        unionTypeName.append("uniontype<");
        for (int i = 0; i < uois.size(); i++) {
            if (i > 0) {
                unionTypeName.append(",");
            }
            unionTypeName.append(uois.get(i).getTypeName());
        }
        unionTypeName.append(">");
        Assert.assertEquals(unionTypeName.toString(), uoi1.getTypeName());
        // TypeInfo
        TypeInfo typeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi1);
        Assert.assertEquals(Category.UNION, typeInfo1.getCategory());
        Assert.assertEquals(UnionTypeInfo.class.getName(), typeInfo1.getClass().getName());
        Assert.assertEquals(typeInfo1.getTypeName(), uoi1.getTypeName());
        Assert.assertEquals(typeInfo1, TypeInfoUtils.getTypeInfoFromTypeString(uoi1.getTypeName()));
        TypeInfo typeInfo2 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi2);
        Assert.assertEquals(typeInfo1, typeInfo2);
        Assert.assertEquals(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo2));
        Assert.assertEquals(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo2));
        // null
        Assert.assertNull(uoi1.getField(null));
        Assert.assertEquals(-1, uoi1.getTag(null));
        // Union
        UnionObject union = new StandardUnionObjectInspector.StandardUnion((byte) 0, 1);
        Assert.assertEquals(0, uoi1.getTag(union));
        Assert.assertEquals(1, uoi1.getField(union));
        Assert.assertEquals("{0:1}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 0, 1), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(1));
        union = new StandardUnionObjectInspector.StandardUnion((byte) 1, "two");
        Assert.assertEquals(1, uoi1.getTag(union));
        Assert.assertEquals("two", uoi1.getField(union));
        Assert.assertEquals("{1:\"two\"}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 1, "two"), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals("two"));
        union = new StandardUnionObjectInspector.StandardUnion((byte) 2, true);
        Assert.assertEquals(2, uoi1.getTag(union));
        Assert.assertEquals(true, uoi1.getField(union));
        Assert.assertEquals("{2:true}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 2, true), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(true));
        ArrayList<Integer> iList = new ArrayList<>();
        iList.add(4);
        iList.add(5);
        union = new StandardUnionObjectInspector.StandardUnion((byte) 3, iList);
        Assert.assertEquals(3, uoi1.getTag(union));
        Assert.assertEquals(iList, uoi1.getField(union));
        Assert.assertEquals("{3:[4,5]}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 3, iList.clone()), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(iList));
        HashMap<Integer, String> map = new HashMap<>();
        map.put(6, "six");
        map.put(7, "seven");
        map.put(8, "eight");
        union = new StandardUnionObjectInspector.StandardUnion((byte) 4, map);
        Assert.assertEquals(4, uoi1.getTag(union));
        Assert.assertEquals(map, uoi1.getField(union));
        Assert.assertEquals("{4:{6:\"six\",7:\"seven\",8:\"eight\"}}", SerDeUtils.getJSONString(union, uoi1));
        Throwable th = null;
        try {
            ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 4, map.clone()), uoi2, null);
        } catch (Throwable t) {
            th = t;
        }
        Assert.assertNotNull(th);
        Assert.assertEquals("Compare on map type not supported!", th.getMessage());
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(map));
        ArrayList<Object> struct = new ArrayList<>(2);
        struct.add(9.0);
        struct.add(10L);
        union = new StandardUnionObjectInspector.StandardUnion((byte) 5, struct);
        Assert.assertEquals(5, uoi1.getTag(union));
        Assert.assertEquals(struct, uoi1.getField(union));
        Assert.assertEquals("{5:{\"mydouble\":9.0,\"mylong\":10}}", SerDeUtils.getJSONString(union, uoi1));
        Assert.assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnionObjectInspector.StandardUnion((byte) 5, struct.clone()), uoi2));
        Assert.assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(struct));
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 2 with UnionObject

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObject in project hive by apache.

the class GroupByOperator method updateAggregations.

/*
   * Update aggregations. If the aggregation is for distinct, in case of hash
   * aggregation, the client tells us whether it is a new entry. For sort-based
   * aggregations, the last row is compared with the current one to figure out
   * whether it has changed. As a cleanup, the lastInvoke logic can be pushed in
   * the caller, and this function can be independent of that. The client should
   * always notify whether it is a different row or not.
   *
   * @param aggs the aggregations to be evaluated
   *
   * @param row the row being processed
   *
   * @param rowInspector the inspector for the row
   *
   * @param hashAggr whether hash aggregation is being performed or not
   *
   * @param newEntryForHashAggr only valid if it is a hash aggregation, whether
   * it is a new entry or not
   */
protected void updateAggregations(AggregationBuffer[] aggs, Object row, ObjectInspector rowInspector, boolean hashAggr, boolean newEntryForHashAggr, Object[][] lastInvoke) throws HiveException {
    if (unionExprEval == null) {
        for (int ai = 0; ai < aggs.length; ai++) {
            // Calculate the parameters
            Object[] o = new Object[aggregationParameterFields[ai].length];
            for (int pi = 0; pi < aggregationParameterFields[ai].length; pi++) {
                o[pi] = aggregationParameterFields[ai][pi].evaluate(row);
            }
            // Update the aggregations.
            if (aggregationIsDistinct[ai]) {
                if (hashAggr) {
                    if (newEntryForHashAggr) {
                        aggregationEvaluators[ai].aggregate(aggs[ai], o);
                    }
                } else {
                    if (lastInvoke[ai] == null) {
                        lastInvoke[ai] = new Object[o.length];
                    }
                    if (ObjectInspectorUtils.compare(o, aggregationParameterObjectInspectors[ai], lastInvoke[ai], aggregationParameterStandardObjectInspectors[ai]) != 0) {
                        aggregationEvaluators[ai].aggregate(aggs[ai], o);
                        for (int pi = 0; pi < o.length; pi++) {
                            lastInvoke[ai][pi] = ObjectInspectorUtils.copyToStandardObject(o[pi], aggregationParameterObjectInspectors[ai][pi], ObjectInspectorCopyOption.WRITABLE);
                        }
                    }
                }
            } else {
                aggregationEvaluators[ai].aggregate(aggs[ai], o);
            }
        }
        return;
    }
    if (distinctKeyAggrs.size() > 0) {
        // evaluate union object
        UnionObject uo = (UnionObject) (unionExprEval.evaluate(row));
        int unionTag = uo.getTag();
        // update non-distinct key aggregations : "KEY._colx:t._coly"
        if (nonDistinctKeyAggrs.get(unionTag) != null) {
            for (int pos : nonDistinctKeyAggrs.get(unionTag)) {
                Object[] o = new Object[aggregationParameterFields[pos].length];
                for (int pi = 0; pi < aggregationParameterFields[pos].length; pi++) {
                    o[pi] = aggregationParameterFields[pos][pi].evaluate(row);
                }
                aggregationEvaluators[pos].aggregate(aggs[pos], o);
            }
        }
        // update them all.
        if (distinctKeyAggrs.get(unionTag) != null) {
            for (int i : distinctKeyAggrs.get(unionTag)) {
                Object[] o = new Object[aggregationParameterFields[i].length];
                for (int pi = 0; pi < aggregationParameterFields[i].length; pi++) {
                    o[pi] = aggregationParameterFields[i][pi].evaluate(row);
                }
                if (hashAggr) {
                    if (newEntryForHashAggr) {
                        aggregationEvaluators[i].aggregate(aggs[i], o);
                    }
                } else {
                    if (lastInvoke[i] == null) {
                        lastInvoke[i] = new Object[o.length];
                    }
                    if (ObjectInspectorUtils.compare(o, aggregationParameterObjectInspectors[i], lastInvoke[i], aggregationParameterStandardObjectInspectors[i]) != 0) {
                        aggregationEvaluators[i].aggregate(aggs[i], o);
                        for (int pi = 0; pi < o.length; pi++) {
                            lastInvoke[i][pi] = ObjectInspectorUtils.copyToStandardObject(o[pi], aggregationParameterObjectInspectors[i][pi], ObjectInspectorCopyOption.WRITABLE);
                        }
                    }
                }
            }
        }
        // these aggregations should be updated only once.
        if (unionTag == 0) {
            for (int pos : nonDistinctAggrs) {
                Object[] o = new Object[aggregationParameterFields[pos].length];
                for (int pi = 0; pi < aggregationParameterFields[pos].length; pi++) {
                    o[pi] = aggregationParameterFields[pos][pi].evaluate(row);
                }
                aggregationEvaluators[pos].aggregate(aggs[pos], o);
            }
        }
    } else {
        for (int ai = 0; ai < aggs.length; ai++) {
            // there is no distinct aggregation,
            // update all aggregations
            Object[] o = new Object[aggregationParameterFields[ai].length];
            for (int pi = 0; pi < aggregationParameterFields[ai].length; pi++) {
                o[pi] = aggregationParameterFields[ai][pi].evaluate(row);
            }
            aggregationEvaluators[ai].aggregate(aggs[ai], o);
        }
    }
}

Example 3 with UnionObject

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObject in project hive by apache.

the class VerifyLazy method lazyCompare.

public static boolean lazyCompare(TypeInfo typeInfo, Object lazyObject, Object expectedObject) {
    if (expectedObject == null) {
        if (lazyObject != null) {
            throw new RuntimeException("Expected object is null but object is not null " + lazyObject.toString() + " typeInfo " + typeInfo.toString());
        }
        return true;
    } else if (lazyObject == null) {
        throw new RuntimeException("Expected object is not null \"" + expectedObject.toString() + "\" typeInfo " + typeInfo.toString() + " but object is null");
    }
    if (lazyObject instanceof Writable) {
        if (!lazyObject.equals(expectedObject)) {
            throw new RuntimeException("Expected object " + expectedObject.toString() + " and actual object " + lazyObject.toString() + " is not equal typeInfo " + typeInfo.toString());
        }
        return true;
    }
    if (lazyObject instanceof LazyPrimitive) {
        Object primitiveObject = ((LazyPrimitive) lazyObject).getObject();
        PrimitiveTypeInfo primitiveTypeInfo = (PrimitiveTypeInfo) typeInfo;
        switch(primitiveTypeInfo.getPrimitiveCategory()) {
            case BOOLEAN:
                {
                    if (!(primitiveObject instanceof LazyBoolean)) {
                        throw new RuntimeException("Expected LazyBoolean");
                    }
                    boolean value = ((LazyBoolean) primitiveObject).getWritableObject().get();
                    boolean expected = ((BooleanWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Boolean field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BYTE:
                {
                    if (!(primitiveObject instanceof LazyByte)) {
                        throw new RuntimeException("Expected LazyByte");
                    }
                    byte value = ((LazyByte) primitiveObject).getWritableObject().get();
                    byte expected = ((ByteWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Byte field mismatch (expected " + (int) expected + " found " + (int) value + ")");
                    }
                }
                break;
            case SHORT:
                {
                    if (!(primitiveObject instanceof LazyShort)) {
                        throw new RuntimeException("Expected LazyShort");
                    }
                    short value = ((LazyShort) primitiveObject).getWritableObject().get();
                    short expected = ((ShortWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Short field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INT:
                {
                    if (!(primitiveObject instanceof LazyInteger)) {
                        throw new RuntimeException("Expected LazyInteger");
                    }
                    int value = ((LazyInteger) primitiveObject).getWritableObject().get();
                    int expected = ((IntWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Int field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case LONG:
                {
                    if (!(primitiveObject instanceof LazyLong)) {
                        throw new RuntimeException("Expected LazyLong");
                    }
                    long value = ((LazyLong) primitiveObject).getWritableObject().get();
                    long expected = ((LongWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Long field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case FLOAT:
                {
                    if (!(primitiveObject instanceof LazyFloat)) {
                        throw new RuntimeException("Expected LazyFloat");
                    }
                    float value = ((LazyFloat) primitiveObject).getWritableObject().get();
                    float expected = ((FloatWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Float field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case DOUBLE:
                {
                    if (!(primitiveObject instanceof LazyDouble)) {
                        throw new RuntimeException("Expected LazyDouble");
                    }
                    double value = ((LazyDouble) primitiveObject).getWritableObject().get();
                    double expected = ((DoubleWritable) expectedObject).get();
                    if (value != expected) {
                        throw new RuntimeException("Double field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case STRING:
                {
                    if (!(primitiveObject instanceof LazyString)) {
                        throw new RuntimeException("Text expected writable not Text");
                    }
                    Text value = ((LazyString) primitiveObject).getWritableObject();
                    Text expected = ((Text) expectedObject);
                    if (!value.equals(expected)) {
                        throw new RuntimeException("String field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case CHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveChar)) {
                        throw new RuntimeException("Expected LazyHiveChar");
                    }
                    HiveChar value = ((LazyHiveChar) primitiveObject).getWritableObject().getHiveChar();
                    HiveChar expected = ((HiveCharWritable) expectedObject).getHiveChar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveChar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case VARCHAR:
                {
                    if (!(primitiveObject instanceof LazyHiveVarchar)) {
                        throw new RuntimeException("Expected LazyHiveVarchar");
                    }
                    HiveVarchar value = ((LazyHiveVarchar) primitiveObject).getWritableObject().getHiveVarchar();
                    HiveVarchar expected = ((HiveVarcharWritable) expectedObject).getHiveVarchar();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveVarchar field mismatch (expected '" + expected + "' found '" + value + "')");
                    }
                }
                break;
            case DECIMAL:
                {
                    if (!(primitiveObject instanceof LazyHiveDecimal)) {
                        throw new RuntimeException("Expected LazyDecimal");
                    }
                    HiveDecimal value = ((LazyHiveDecimal) primitiveObject).getWritableObject().getHiveDecimal();
                    HiveDecimal expected = ((HiveDecimalWritable) expectedObject).getHiveDecimal();
                    if (!value.equals(expected)) {
                        DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveTypeInfo;
                        int precision = decimalTypeInfo.getPrecision();
                        int scale = decimalTypeInfo.getScale();
                        throw new RuntimeException("Decimal field mismatch (expected " + expected.toString() + " found " + value.toString() + ") precision " + precision + ", scale " + scale);
                    }
                }
                break;
            case DATE:
                {
                    if (!(primitiveObject instanceof LazyDate)) {
                        throw new RuntimeException("Expected LazyDate");
                    }
                    Date value = ((LazyDate) primitiveObject).getWritableObject().get();
                    Date expected = ((DateWritableV2) expectedObject).get();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Date field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case TIMESTAMP:
                {
                    if (!(primitiveObject instanceof LazyTimestamp)) {
                        throw new RuntimeException("TimestampWritableV2 expected writable not TimestampWritableV2");
                    }
                    Timestamp value = ((LazyTimestamp) primitiveObject).getWritableObject().getTimestamp();
                    Timestamp expected = ((TimestampWritableV2) expectedObject).getTimestamp();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("Timestamp field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_YEAR_MONTH:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalYearMonth)) {
                        throw new RuntimeException("Expected LazyHiveIntervalYearMonth");
                    }
                    HiveIntervalYearMonth value = ((LazyHiveIntervalYearMonth) primitiveObject).getWritableObject().getHiveIntervalYearMonth();
                    HiveIntervalYearMonth expected = ((HiveIntervalYearMonthWritable) expectedObject).getHiveIntervalYearMonth();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalYearMonth field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case INTERVAL_DAY_TIME:
                {
                    if (!(primitiveObject instanceof LazyHiveIntervalDayTime)) {
                        throw new RuntimeException("Expected writable LazyHiveIntervalDayTime");
                    }
                    HiveIntervalDayTime value = ((LazyHiveIntervalDayTime) primitiveObject).getWritableObject().getHiveIntervalDayTime();
                    HiveIntervalDayTime expected = ((HiveIntervalDayTimeWritable) expectedObject).getHiveIntervalDayTime();
                    if (!value.equals(expected)) {
                        throw new RuntimeException("HiveIntervalDayTime field mismatch (expected " + expected + " found " + value + ")");
                    }
                }
                break;
            case BINARY:
                {
                    if (!(primitiveObject instanceof LazyBinary)) {
                        throw new RuntimeException("Expected LazyBinary");
                    }
                    BytesWritable bytesWritable = ((LazyBinary) primitiveObject).getWritableObject();
                    byte[] value = Arrays.copyOfRange(bytesWritable.getBytes(), 0, bytesWritable.getLength());
                    BytesWritable bytesWritableExpected = (BytesWritable) expectedObject;
                    byte[] expected = Arrays.copyOfRange(bytesWritableExpected.getBytes(), 0, bytesWritableExpected.getLength());
                    if (value.length != expected.length) {
                        throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                    }
                    for (int b = 0; b < value.length; b++) {
                        if (value[b] != expected[b]) {
                            throw new RuntimeException("Byte Array field mismatch (expected " + Arrays.toString(expected) + " found " + Arrays.toString(value) + ")");
                        }
                    }
                }
                break;
            default:
                throw new Error("Unknown primitive category " + primitiveTypeInfo.getPrimitiveCategory());
        }
    } else if (lazyObject instanceof LazyArray) {
        LazyArray lazyArray = (LazyArray) lazyObject;
        List<Object> list = lazyArray.getList();
        List<Object> expectedList = (List<Object>) expectedObject;
        ListTypeInfo listTypeInfo = (ListTypeInfo) typeInfo;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")" + " elementTypeInfo " + listTypeInfo.getListElementTypeInfo().toString());
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (typeInfo instanceof ListTypeInfo) {
        List<Object> list;
        if (lazyObject instanceof LazyBinaryArray) {
            list = ((LazyBinaryArray) lazyObject).getList();
        } else {
            list = (List<Object>) lazyObject;
        }
        List<Object> expectedList = (List<Object>) expectedObject;
        if (list.size() != expectedList.size()) {
            throw new RuntimeException("SerDe deserialized list length does not match (list " + list.toString() + " list.size() " + list.size() + " expectedList " + expectedList.toString() + " expectedList.size() " + expectedList.size() + ")");
        }
        return lazyCompareList((ListTypeInfo) typeInfo, list, expectedList);
    } else if (lazyObject instanceof LazyMap) {
        LazyMap lazyMap = (LazyMap) lazyObject;
        Map<Object, Object> map = lazyMap.getMap();
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (typeInfo instanceof MapTypeInfo) {
        Map<Object, Object> map;
        Map<Object, Object> expectedMap = (Map<Object, Object>) expectedObject;
        if (lazyObject instanceof LazyBinaryMap) {
            map = ((LazyBinaryMap) lazyObject).getMap();
        } else {
            map = (Map<Object, Object>) lazyObject;
        }
        return lazyCompareMap((MapTypeInfo) typeInfo, map, expectedMap);
    } else if (lazyObject instanceof LazyStruct) {
        LazyStruct lazyStruct = (LazyStruct) lazyObject;
        List<Object> fields = lazyStruct.getFieldsAsList();
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (typeInfo instanceof StructTypeInfo) {
        ArrayList<Object> fields;
        if (lazyObject instanceof LazyBinaryStruct) {
            fields = ((LazyBinaryStruct) lazyObject).getFieldsAsList();
        } else {
            fields = (ArrayList<Object>) lazyObject;
        }
        List<Object> expectedFields = (List<Object>) expectedObject;
        StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        return lazyCompareStruct(structTypeInfo, fields, expectedFields);
    } else if (lazyObject instanceof LazyUnion) {
        LazyUnion union = (LazyUnion) lazyObject;
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        return lazyCompareUnion(unionTypeInfo, union, expectedUnion);
    } else if (typeInfo instanceof UnionTypeInfo) {
        StandardUnionObjectInspector.StandardUnion expectedUnion = (StandardUnionObjectInspector.StandardUnion) expectedObject;
        UnionTypeInfo unionTypeInfo = (UnionTypeInfo) typeInfo;
        if (lazyObject instanceof LazyBinaryUnion) {
            return lazyCompareUnion(unionTypeInfo, (LazyBinaryUnion) lazyObject, expectedUnion);
        } else {
            return lazyCompareUnion(unionTypeInfo, (UnionObject) lazyObject, expectedUnion);
        }
    } else {
        System.err.println("Not implemented " + typeInfo.getClass().getName());
    }
    return true;
}

Example 4 with UnionObject

use of org.apache.hadoop.hive.serde2.objectinspector.UnionObject in project hive by apache.

the class TestStandardObjectInspectors method testStandardUnionObjectInspector.

@SuppressWarnings("unchecked")
@Test
public void testStandardUnionObjectInspector() throws Throwable {
    try {
        ArrayList<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>();
        // add primitive types
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
        objectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
        // add a list
        objectInspectors.add(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector));
        // add a map
        objectInspectors.add(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector));
        // add a struct
        List<String> fieldNames = new ArrayList<String>();
        fieldNames.add("myDouble");
        fieldNames.add("myLong");
        ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaDoubleObjectInspector);
        fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaLongObjectInspector);
        objectInspectors.add(ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors));
        StandardUnionObjectInspector uoi1 = ObjectInspectorFactory.getStandardUnionObjectInspector(objectInspectors);
        StandardUnionObjectInspector uoi2 = ObjectInspectorFactory.getStandardUnionObjectInspector((ArrayList<ObjectInspector>) objectInspectors.clone());
        assertEquals(uoi1, uoi2);
        assertEquals(ObjectInspectorUtils.getObjectInspectorName(uoi1), ObjectInspectorUtils.getObjectInspectorName(uoi2));
        assertTrue(ObjectInspectorUtils.compareTypes(uoi1, uoi2));
        // compareSupported returns false because Union can contain
        // an object of Map
        assertFalse(ObjectInspectorUtils.compareSupported(uoi1));
        // construct unionObjectInspector without Map field.
        ArrayList<ObjectInspector> ois = (ArrayList<ObjectInspector>) objectInspectors.clone();
        ois.set(4, PrimitiveObjectInspectorFactory.javaIntObjectInspector);
        assertTrue(ObjectInspectorUtils.compareSupported(ObjectInspectorFactory.getStandardUnionObjectInspector(ois)));
        // metadata
        assertEquals(Category.UNION, uoi1.getCategory());
        List<? extends ObjectInspector> uois = uoi1.getObjectInspectors();
        assertEquals(6, uois.size());
        for (int i = 0; i < 6; i++) {
            assertEquals(objectInspectors.get(i), uois.get(i));
        }
        StringBuilder unionTypeName = new StringBuilder();
        unionTypeName.append("uniontype<");
        for (int i = 0; i < uois.size(); i++) {
            if (i > 0) {
                unionTypeName.append(",");
            }
            unionTypeName.append(uois.get(i).getTypeName());
        }
        unionTypeName.append(">");
        assertEquals(unionTypeName.toString(), uoi1.getTypeName());
        // TypeInfo
        TypeInfo typeInfo1 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi1);
        assertEquals(Category.UNION, typeInfo1.getCategory());
        assertEquals(UnionTypeInfo.class.getName(), typeInfo1.getClass().getName());
        assertEquals(typeInfo1.getTypeName(), uoi1.getTypeName());
        assertEquals(typeInfo1, TypeInfoUtils.getTypeInfoFromTypeString(uoi1.getTypeName()));
        TypeInfo typeInfo2 = TypeInfoUtils.getTypeInfoFromObjectInspector(uoi2);
        assertEquals(typeInfo1, typeInfo2);
        assertEquals(TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo2));
        assertEquals(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo1), TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo2));
        // null
        assertNull(uoi1.getField(null));
        assertEquals(-1, uoi1.getTag(null));
        // Union
        UnionObject union = new StandardUnion((byte) 0, 1);
        assertEquals(0, uoi1.getTag(union));
        assertEquals(1, uoi1.getField(union));
        assertEquals("{0:1}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 0, 1), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        union = new StandardUnion((byte) 1, "two");
        assertEquals(1, uoi1.getTag(union));
        assertEquals("two", uoi1.getField(union));
        assertEquals("{1:\"two\"}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 1, "two"), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        union = new StandardUnion((byte) 2, true);
        assertEquals(2, uoi1.getTag(union));
        assertEquals(true, uoi1.getField(union));
        assertEquals("{2:true}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 2, true), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        ArrayList<Integer> iList = new ArrayList<Integer>();
        iList.add(4);
        iList.add(5);
        union = new StandardUnion((byte) 3, iList);
        assertEquals(3, uoi1.getTag(union));
        assertEquals(iList, uoi1.getField(union));
        assertEquals("{3:[4,5]}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 3, iList.clone()), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        HashMap<Integer, String> map = new LinkedHashMap<Integer, String>();
        map.put(6, "six");
        map.put(7, "seven");
        map.put(8, "eight");
        union = new StandardUnion((byte) 4, map);
        assertEquals(4, uoi1.getTag(union));
        assertEquals(map, uoi1.getField(union));
        assertEquals("{4:{6:\"six\",7:\"seven\",8:\"eight\"}}", SerDeUtils.getJSONString(union, uoi1));
        Throwable th = null;
        try {
            ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 4, map.clone()), uoi2, null);
        } catch (Throwable t) {
            th = t;
        }
        assertNotNull(th);
        assertEquals("Compare on map type not supported!", th.getMessage());
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
        ArrayList<Object> struct = new ArrayList<Object>(2);
        struct.add(9.0);
        struct.add(10L);
        union = new StandardUnion((byte) 5, struct);
        assertEquals(5, uoi1.getTag(union));
        assertEquals(struct, uoi1.getField(union));
        assertEquals("{5:{\"mydouble\":9.0,\"mylong\":10}}", SerDeUtils.getJSONString(union, uoi1));
        assertEquals(0, ObjectInspectorUtils.compare(union, uoi1, new StandardUnion((byte) 5, struct.clone()), uoi2));
        assertTrue(ObjectInspectorUtils.copyToStandardObject(union, uoi1).equals(union));
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}