Examples with StructObjectInspector org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector used on opensource projects

Example 36 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class TestSimpleMapEqualComparer method serializeAndDeserialize.

Object serializeAndDeserialize(TextStringMapHolder o1, StructObjectInspector oi1, LazySimpleSerDe serde, LazySerDeParameters serdeParams) throws IOException, SerDeException {
    ByteStream.Output serializeStream = new ByteStream.Output();
    LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams.getSeparators(), 0, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(), serdeParams.getNeedsEscape());
    Text t = new Text(serializeStream.toByteArray());
    return serde.deserialize(t);
}

Example 37 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class TestSimpleMapEqualComparer method testCompatibleType.

public void testCompatibleType() throws SerDeException, IOException {
    // empty maps
    TextStringMapHolder o1 = new TextStringMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory.getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);
    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    LazySerDeParameters serdeParams = new LazySerDeParameters(conf, tbl, LazySimpleSerDe.class.getName());
    SerDeUtils.initializeSerDe(serde, conf, tbl, null);
    ObjectInspector oi2 = serde.getObjectInspector();
    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertEquals(0, rc);
    // equal maps
    o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything");
    o1.mMap.put(new Text("1729"), "A taxi cab number");
    o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertEquals(0, rc);
    // unequal maps
    o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number");
    rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertFalse(0 == rc);
}

Example 38 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class TestThriftObjectInspectors method testThriftObjectInspectors.

public void testThriftObjectInspectors() throws Throwable {
    try {
        ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        assertEquals(oi1, oi2);
        // metadata
        assertEquals(Category.STRUCT, oi1.getCategory());
        StructObjectInspector soi = (StructObjectInspector) oi1;
        List<? extends StructField> fields = soi.getAllStructFieldRefs();
        assertEquals(10, fields.size());
        assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
        // null
        for (int i = 0; i < fields.size(); i++) {
            assertNull(soi.getStructFieldData(null, fields.get(i)));
        }
        ObjectInspector oi = ObjectInspectorFactory.getReflectionObjectInspector(PropValueUnion.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
        assertNotNull(oi.toString());
        // real object
        Complex c = new Complex();
        c.setAint(1);
        c.setAString("test");
        List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
        c.setLint(c2);
        List<String> c3 = Arrays.asList(new String[] { "one", "two" });
        c.setLString(c3);
        List<IntString> c4 = new ArrayList<IntString>();
        c.setLintString(c4);
        c.setMStringString(null);
        c.setAttributes(null);
        c.setUnionField1(null);
        c.setUnionField2(null);
        c.setUnionField3(null);
        assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
        assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
        assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
        assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
        assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
        assertNull(soi.getStructFieldData(c, fields.get(5)));
        assertNull(soi.getStructFieldData(c, fields.get(6)));
        assertNull(soi.getStructFieldData(c, fields.get(7)));
        assertNull(soi.getStructFieldData(c, fields.get(8)));
        assertNull(soi.getStructFieldData(c, fields.get(9)));
        ArrayList<Object> cfields = new ArrayList<Object>();
        for (int i = 0; i < 10; i++) {
            cfields.add(soi.getStructFieldData(c, fields.get(i)));
        }
        assertEquals(cfields, soi.getStructFieldsDataAsList(c));
        // sub fields
        assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
        assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)), fields.get(4).getFieldObjectInspector());
        assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(5).getFieldObjectInspector());
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}

Example 39 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class GenericUDFSortArrayByField method initialize.

@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    GenericUDFUtils.ReturnObjectInspectorResolver returnOIResolver;
    returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
    /**This UDF requires minimum 2 arguments array_name,field name*/
    if (arguments.length < 2) {
        throw new UDFArgumentLengthException("SORT_ARRAY_BY requires minimum 2 arguments, got " + arguments.length);
    }
    /**First argument must be array*/
    switch(arguments[0].getCategory()) {
        case LIST:
            listObjectInspector = (ListObjectInspector) arguments[0];
            break;
        default:
            throw new UDFArgumentTypeException(0, "Argument 1 of function SORT_ARRAY_BY must be " + serdeConstants.LIST_TYPE_NAME + ", but " + arguments[0].getTypeName() + " was found.");
    }
    /**Elements inside first argument(array) must be tuple(s)*/
    switch(listObjectInspector.getListElementObjectInspector().getCategory()) {
        case STRUCT:
            structObjectInspector = (StructObjectInspector) listObjectInspector.getListElementObjectInspector();
            break;
        default:
            throw new UDFArgumentTypeException(0, "Element[s] of first argument array in function SORT_ARRAY_BY must be " + serdeConstants.STRUCT_TYPE_NAME + ", but " + listObjectInspector.getTypeName() + " was found.");
    }
    /**All sort fields argument name and sort order name must be in String type*/
    converters = new Converter[arguments.length];
    inputTypes = new PrimitiveCategory[arguments.length];
    fields = new StructField[arguments.length - 1];
    noOfInputFields = arguments.length - 1;
    for (int i = 1; i < arguments.length; i++) {
        checkArgPrimitive(arguments, i);
        checkArgGroups(arguments, i, inputTypes, PrimitiveGrouping.STRING_GROUP);
        if (arguments[i] instanceof ConstantObjectInspector) {
            String fieldName = getConstantStringValue(arguments, i);
            /**checking whether any sorting order (ASC,DESC) has specified in last argument*/
            if (i != 1 && (i == arguments.length - 1) && (fieldName.trim().toUpperCase().equals(SORT_ORDER_TYPE.ASC.name()) || fieldName.trim().toUpperCase().equals(SORT_ORDER_TYPE.DESC.name()))) {
                sortOrder = SORT_ORDER_TYPE.valueOf(fieldName.trim().toUpperCase());
                noOfInputFields -= 1;
                continue;
            }
            fields[i - 1] = structObjectInspector.getStructFieldRef(getConstantStringValue(arguments, i));
        }
        obtainStringConverter(arguments, i, inputTypes, converters);
    }
    ObjectInspector returnOI = returnOIResolver.get(structObjectInspector);
    converters[0] = ObjectInspectorConverters.getConverter(structObjectInspector, returnOI);
    return ObjectInspectorFactory.getStandardListObjectInspector(structObjectInspector);
}

Example 40 with StructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector in project hive by apache.

the class TestOperators method testScriptOperator.

public void testScriptOperator() throws Throwable {
    try {
        System.out.println("Testing Script Operator");
        // col1
        ExprNodeDesc exprDesc1 = TestExecDriver.getStringColumn("col1");
        // col2
        ExprNodeDesc expr1 = TestExecDriver.getStringColumn("col0");
        ExprNodeDesc expr2 = new ExprNodeConstantDesc("1");
        ExprNodeDesc exprDesc2 = TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc("concat", expr1, expr2);
        // select operator to project these two columns
        ArrayList<ExprNodeDesc> earr = new ArrayList<ExprNodeDesc>();
        earr.add(exprDesc1);
        earr.add(exprDesc2);
        ArrayList<String> outputCols = new ArrayList<String>();
        for (int i = 0; i < earr.size(); i++) {
            outputCols.add("_col" + i);
        }
        SelectDesc selectCtx = new SelectDesc(earr, outputCols);
        Operator<SelectDesc> op = OperatorFactory.get(new CompilationOpContext(), SelectDesc.class);
        op.setConf(selectCtx);
        // scriptOperator to echo the output of the select
        TableDesc scriptOutput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b");
        TableDesc scriptInput = PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "a,b");
        ScriptDesc sd = new ScriptDesc("cat", scriptOutput, TextRecordWriter.class, scriptInput, TextRecordReader.class, TextRecordReader.class, PlanUtils.getDefaultTableDesc("" + Utilities.tabCode, "key"));
        Operator<ScriptDesc> sop = OperatorFactory.getAndMakeChild(sd, op);
        // Collect operator to observe the output of the script
        CollectDesc cd = new CollectDesc(Integer.valueOf(10));
        CollectOperator cdop = (CollectOperator) OperatorFactory.getAndMakeChild(cd, sop);
        op.initialize(new JobConf(TestOperators.class), new ObjectInspector[] { r[0].oi });
        // evaluate on row
        for (int i = 0; i < 5; i++) {
            op.process(r[i].o, 0);
        }
        op.close(false);
        InspectableObject io = new InspectableObject();
        for (int i = 0; i < 5; i++) {
            cdop.retrieve(io);
            System.out.println("[" + i + "] io.o=" + io.o);
            System.out.println("[" + i + "] io.oi=" + io.oi);
            StructObjectInspector soi = (StructObjectInspector) io.oi;
            assert (soi != null);
            StructField a = soi.getStructFieldRef("a");
            StructField b = soi.getStructFieldRef("b");
            assertEquals("" + (i + 1), ((PrimitiveObjectInspector) a.getFieldObjectInspector()).getPrimitiveJavaObject(soi.getStructFieldData(io.o, a)));
            assertEquals((i) + "1", ((PrimitiveObjectInspector) b.getFieldObjectInspector()).getPrimitiveJavaObject(soi.getStructFieldData(io.o, b)));
        }
        System.out.println("Script Operator ok");
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    }
}