Examples with StandardStructObjectInspector org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector used on opensource projects

Example 11 with StandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector in project cdap by caskdata.

the class StandardObjectInspectorsTest method doStandardObjectInspectorTest.

private void doStandardObjectInspectorTest(boolean testComments) {
    ArrayList<String> fieldNames = new ArrayList<>();
    fieldNames.add("firstInteger");
    fieldNames.add("secondString");
    fieldNames.add("thirdBoolean");
    ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<>();
    fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    ArrayList<String> fieldComments = new ArrayList<>(3);
    if (testComments) {
        fieldComments.add("firstInteger comment");
        fieldComments.add("secondString comment");
        fieldComments.add("thirdBoolean comment");
    } else {
        // should have null for non-specified comments
        for (int i = 0; i < 3; i++) {
            fieldComments.add(null);
        }
    }
    StandardStructObjectInspector soi1 = testComments ? ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors, fieldComments) : ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
    StandardStructObjectInspector soi2 = testComments ? ObjectInspectorFactory.getStandardStructObjectInspector((ArrayList<String>) fieldNames.clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors.clone(), (ArrayList<String>) fieldComments.clone()) : ObjectInspectorFactory.getStandardStructObjectInspector((ArrayList<String>) fieldNames.clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors.clone());
    Assert.assertEquals(soi1, soi2);
    // metadata
    Assert.assertEquals(Category.STRUCT, soi1.getCategory());
    List<? extends StructField> fields = soi1.getAllStructFieldRefs();
    Assert.assertEquals(3, fields.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i).getFieldName());
        Assert.assertEquals(fieldObjectInspectors.get(i), fields.get(i).getFieldObjectInspector());
        Assert.assertEquals(fieldComments.get(i), fields.get(i).getFieldComment());
    }
    Assert.assertEquals(fields.get(1), soi1.getStructFieldRef("secondString"));
    StringBuilder structTypeName = new StringBuilder();
    structTypeName.append("struct<");
    for (int i = 0; i < fields.size(); i++) {
        if (i > 0) {
            structTypeName.append(",");
        }
        structTypeName.append(fields.get(i).getFieldName());
        structTypeName.append(":");
        structTypeName.append(fields.get(i).getFieldObjectInspector().getTypeName());
    }
    structTypeName.append(">");
    Assert.assertEquals(structTypeName.toString(), soi1.getTypeName());
    // null
    Assert.assertNull(soi1.getStructFieldData(null, fields.get(0)));
    Assert.assertNull(soi1.getStructFieldData(null, fields.get(1)));
    Assert.assertNull(soi1.getStructFieldData(null, fields.get(2)));
    Assert.assertNull(soi1.getStructFieldsDataAsList(null));
    // HashStruct
    ArrayList<Object> struct = new ArrayList<>(3);
    struct.add(1);
    struct.add("two");
    struct.add(true);
    Assert.assertEquals(1, soi1.getStructFieldData(struct, fields.get(0)));
    Assert.assertEquals("two", soi1.getStructFieldData(struct, fields.get(1)));
    Assert.assertEquals(true, soi1.getStructFieldData(struct, fields.get(2)));
    // Settable
    Object struct3 = soi1.create();
    soi1.setStructFieldData(struct3, fields.get(0), 1);
    soi1.setStructFieldData(struct3, fields.get(1), "two");
    soi1.setStructFieldData(struct3, fields.get(2), true);
    Assert.assertEquals(struct, struct3);
}

Example 12 with StandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector in project hive by apache.

the class MyTestPrimitiveClass method getRowInspector.

public StructObjectInspector getRowInspector(PrimitiveTypeInfo[] primitiveTypeInfos) {
    List<String> columnNames = new ArrayList<String>(primitiveCount);
    List<ObjectInspector> primitiveObjectInspectorList = new ArrayList<ObjectInspector>(primitiveCount);
    for (int index = 0; index < MyTestPrimitiveClass.primitiveCount; index++) {
        columnNames.add(String.format("col%d", index));
        PrimitiveTypeInfo primitiveTypeInfo = primitiveTypeInfos[index];
        PrimitiveCategory primitiveCategory = primitiveTypeInfo.getPrimitiveCategory();
        primitiveObjectInspectorList.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(primitiveCategory));
    }
    StandardStructObjectInspector rowOI = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, primitiveObjectInspectorList);
    return rowOI;
}

Example 13 with StandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector in project hive by apache.

the class TestAvroDeserializer method canDeserializeVoidType.

@Test
public void canDeserializeVoidType() throws IOException, SerDeException {
    String schemaString = "{\n" + "  \"type\": \"record\", \n" + "  \"name\": \"nullTest\",\n" + "  \"fields\" : [\n" + "    {\"name\": \"isANull\", \"type\": \"null\"}\n" + "  ]\n" + "}";
    Schema s = AvroSerdeUtils.getSchemaFor(schemaString);
    GenericData.Record record = new GenericData.Record(s);
    record.put("isANull", null);
    assertTrue(GENERIC_DATA.validate(s, record));
    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
    AvroDeserializer de = new AvroDeserializer();
    ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
    assertEquals(1, row.size());
    Object theVoidObject = row.get(0);
    assertNull(theVoidObject);
    StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
    StructField fieldRef = oi.getStructFieldRef("isANull");
    Object shouldBeNull = oi.getStructFieldData(row, fieldRef);
    assertNull(shouldBeNull);
    assertTrue(fieldRef.getFieldObjectInspector() instanceof VoidObjectInspector);
}

Example 14 with StandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector in project hive by apache.

the class TestAvroDeserializer method canDeserializeRecordsInternal.

public void canDeserializeRecordsInternal(Schema s, Schema fileSchema) throws SerDeException, IOException {
    GenericData.Record record = new GenericData.Record(s);
    GenericData.Record innerRecord = new GenericData.Record(s.getField("aRecord").schema());
    innerRecord.put("int1", 42);
    innerRecord.put("boolean1", true);
    innerRecord.put("long1", 42432234234l);
    record.put("aRecord", innerRecord);
    assertTrue(GENERIC_DATA.validate(s, record));
    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record, fileSchema);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
    AvroDeserializer de = new AvroDeserializer();
    ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
    assertEquals(1, row.size());
    Object theRecordObject = row.get(0);
    System.out.println("theRecordObject = " + theRecordObject.getClass().getCanonicalName());
    // The original record was lost in the deserialization, so just go the
    // correct way, through objectinspectors
    StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
    List<? extends StructField> allStructFieldRefs = oi.getAllStructFieldRefs();
    assertEquals(1, allStructFieldRefs.size());
    StructField fieldRefForaRecord = allStructFieldRefs.get(0);
    assertEquals("arecord", fieldRefForaRecord.getFieldName());
    Object innerRecord2 = oi.getStructFieldData(row, fieldRefForaRecord);
    // Extract innerRecord field refs
    StandardStructObjectInspector innerRecord2OI = (StandardStructObjectInspector) fieldRefForaRecord.getFieldObjectInspector();
    List<? extends StructField> allStructFieldRefs1 = innerRecord2OI.getAllStructFieldRefs();
    assertEquals(3, allStructFieldRefs1.size());
    assertEquals("int1", allStructFieldRefs1.get(0).getFieldName());
    assertEquals("boolean1", allStructFieldRefs1.get(1).getFieldName());
    assertEquals("long1", allStructFieldRefs1.get(2).getFieldName());
    innerRecord2OI.getStructFieldsDataAsList(innerRecord2);
    assertEquals(42, innerRecord2OI.getStructFieldData(innerRecord2, allStructFieldRefs1.get(0)));
    assertEquals(true, innerRecord2OI.getStructFieldData(innerRecord2, allStructFieldRefs1.get(1)));
    assertEquals(42432234234l, innerRecord2OI.getStructFieldData(innerRecord2, allStructFieldRefs1.get(2)));
}

Example 15 with StandardStructObjectInspector

use of org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector in project hive by apache.

the class TestAvroDeserializer method canDeserializeMapsWithPrimitiveKeys.

@Test
public void canDeserializeMapsWithPrimitiveKeys() throws SerDeException, IOException {
    Schema s = AvroSerdeUtils.getSchemaFor(TestAvroObjectInspectorGenerator.MAP_WITH_PRIMITIVE_VALUE_TYPE);
    GenericData.Record record = new GenericData.Record(s);
    Map<String, Long> m = new Hashtable<String, Long>();
    m.put("one", 1l);
    m.put("two", 2l);
    m.put("three", 3l);
    record.put("aMap", m);
    assertTrue(GENERIC_DATA.validate(s, record));
    System.out.println("record = " + record);
    AvroGenericRecordWritable garw = Utils.serializeAndDeserializeRecord(record);
    AvroObjectInspectorGenerator aoig = new AvroObjectInspectorGenerator(s);
    AvroDeserializer de = new AvroDeserializer();
    ArrayList<Object> row = (ArrayList<Object>) de.deserialize(aoig.getColumnNames(), aoig.getColumnTypes(), garw, s);
    assertEquals(1, row.size());
    Object theMapObject = row.get(0);
    assertTrue(theMapObject instanceof Map);
    Map theMap = (Map) theMapObject;
    // Verify the raw object that's been created
    assertEquals(1l, theMap.get("one"));
    assertEquals(2l, theMap.get("two"));
    assertEquals(3l, theMap.get("three"));
    // Verify that the provided object inspector can pull out these same values
    StandardStructObjectInspector oi = (StandardStructObjectInspector) aoig.getObjectInspector();
    List<Object> z = oi.getStructFieldsDataAsList(row);
    assertEquals(1, z.size());
    StructField fieldRef = oi.getStructFieldRef("amap");
    Map theMap2 = (Map) oi.getStructFieldData(row, fieldRef);
    assertEquals(1l, theMap2.get("one"));
    assertEquals(2l, theMap2.get("two"));
    assertEquals(3l, theMap2.get("three"));
}