Examples with StructField org.apache.hadoop.hive.serde2.objectinspector.StructField used on opensource projects

Example 51 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project presto by prestodb.

the class OrcTester method hasType.

private static boolean hasType(ObjectInspector objectInspector, PrimitiveCategory... types) {
    if (objectInspector instanceof PrimitiveObjectInspector) {
        PrimitiveObjectInspector primitiveInspector = (PrimitiveObjectInspector) objectInspector;
        PrimitiveCategory primitiveCategory = primitiveInspector.getPrimitiveCategory();
        for (PrimitiveCategory type : types) {
            if (primitiveCategory == type) {
                return true;
            }
        }
        return false;
    }
    if (objectInspector instanceof ListObjectInspector) {
        ListObjectInspector listInspector = (ListObjectInspector) objectInspector;
        return hasType(listInspector.getListElementObjectInspector(), types);
    }
    if (objectInspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) objectInspector;
        return hasType(mapInspector.getMapKeyObjectInspector(), types) || hasType(mapInspector.getMapValueObjectInspector(), types);
    }
    if (objectInspector instanceof StructObjectInspector) {
        for (StructField field : ((StructObjectInspector) objectInspector).getAllStructFieldRefs()) {
            if (hasType(field.getFieldObjectInspector(), types)) {
                return true;
            }
        }
        return false;
    }
    throw new IllegalArgumentException("Unknown object inspector type " + objectInspector);
}

Example 52 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project cdap by caskdata.

the class ObjectInspectorFactoryTest method assertObjectInspection.

private void assertObjectInspection(Type t, Object data) throws Exception {
    Field[] tmpFields;
    // Build the expected fields, based on the type t. Exclude the transient fields.
    if (t instanceof ParameterizedType) {
        ParameterizedType pt = (ParameterizedType) t;
        // TODO either test getDeclaredNonStaticFields or use another method
        tmpFields = ObjectInspectorUtils.getDeclaredNonStaticFields((Class<?>) pt.getRawType());
    } else {
        tmpFields = ObjectInspectorUtils.getDeclaredNonStaticFields((Class<?>) t);
    }
    ImmutableList.Builder builder = ImmutableList.builder();
    for (Field f : tmpFields) {
        if (!Modifier.isTransient(f.getModifiers()) && !f.isSynthetic()) {
            builder.add(f);
        }
    }
    List<Field> expectedFields = builder.build();
    ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(t);
    ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(t);
    Assert.assertEquals(oi1, oi2);
    // metadata
    Assert.assertEquals(ObjectInspector.Category.STRUCT, oi1.getCategory());
    StructObjectInspector soi = (StructObjectInspector) oi1;
    List<? extends StructField> inspectorFields = soi.getAllStructFieldRefs();
    Assert.assertEquals(expectedFields.size(), inspectorFields.size());
    // null
    for (int i = 0; i < inspectorFields.size(); i++) {
        Assert.assertNull(soi.getStructFieldData(null, inspectorFields.get(i)));
    }
    Assert.assertNull(soi.getStructFieldsDataAsList(null));
    // non nulls
    ArrayList<Object> afields = new ArrayList<>();
    for (int i = 0; i < expectedFields.size(); i++) {
        Assert.assertEquals(expectedFields.get(i).get(data), soi.getStructFieldData(data, inspectorFields.get(i)));
        afields.add(soi.getStructFieldData(data, inspectorFields.get(i)));
    }
    Assert.assertEquals(afields, soi.getStructFieldsDataAsList(data));
}

Example 53 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project cdap by caskdata.

the class StandardObjectInspectorsTest method doStandardObjectInspectorTest.

private void doStandardObjectInspectorTest(boolean testComments) {
    ArrayList<String> fieldNames = new ArrayList<>();
    fieldNames.add("firstInteger");
    fieldNames.add("secondString");
    fieldNames.add("thirdBoolean");
    ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<>();
    fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    fieldObjectInspectors.add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    ArrayList<String> fieldComments = new ArrayList<>(3);
    if (testComments) {
        fieldComments.add("firstInteger comment");
        fieldComments.add("secondString comment");
        fieldComments.add("thirdBoolean comment");
    } else {
        // should have null for non-specified comments
        for (int i = 0; i < 3; i++) {
            fieldComments.add(null);
        }
    }
    StandardStructObjectInspector soi1 = testComments ? ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors, fieldComments) : ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
    StandardStructObjectInspector soi2 = testComments ? ObjectInspectorFactory.getStandardStructObjectInspector((ArrayList<String>) fieldNames.clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors.clone(), (ArrayList<String>) fieldComments.clone()) : ObjectInspectorFactory.getStandardStructObjectInspector((ArrayList<String>) fieldNames.clone(), (ArrayList<ObjectInspector>) fieldObjectInspectors.clone());
    Assert.assertEquals(soi1, soi2);
    // metadata
    Assert.assertEquals(Category.STRUCT, soi1.getCategory());
    List<? extends StructField> fields = soi1.getAllStructFieldRefs();
    Assert.assertEquals(3, fields.size());
    for (int i = 0; i < 3; i++) {
        Assert.assertEquals(fieldNames.get(i).toLowerCase(), fields.get(i).getFieldName());
        Assert.assertEquals(fieldObjectInspectors.get(i), fields.get(i).getFieldObjectInspector());
        Assert.assertEquals(fieldComments.get(i), fields.get(i).getFieldComment());
    }
    Assert.assertEquals(fields.get(1), soi1.getStructFieldRef("secondString"));
    StringBuilder structTypeName = new StringBuilder();
    structTypeName.append("struct<");
    for (int i = 0; i < fields.size(); i++) {
        if (i > 0) {
            structTypeName.append(",");
        }
        structTypeName.append(fields.get(i).getFieldName());
        structTypeName.append(":");
        structTypeName.append(fields.get(i).getFieldObjectInspector().getTypeName());
    }
    structTypeName.append(">");
    Assert.assertEquals(structTypeName.toString(), soi1.getTypeName());
    // null
    Assert.assertNull(soi1.getStructFieldData(null, fields.get(0)));
    Assert.assertNull(soi1.getStructFieldData(null, fields.get(1)));
    Assert.assertNull(soi1.getStructFieldData(null, fields.get(2)));
    Assert.assertNull(soi1.getStructFieldsDataAsList(null));
    // HashStruct
    ArrayList<Object> struct = new ArrayList<>(3);
    struct.add(1);
    struct.add("two");
    struct.add(true);
    Assert.assertEquals(1, soi1.getStructFieldData(struct, fields.get(0)));
    Assert.assertEquals("two", soi1.getStructFieldData(struct, fields.get(1)));
    Assert.assertEquals(true, soi1.getStructFieldData(struct, fields.get(2)));
    // Settable
    Object struct3 = soi1.create();
    soi1.setStructFieldData(struct3, fields.get(0), 1);
    soi1.setStructFieldData(struct3, fields.get(1), "two");
    soi1.setStructFieldData(struct3, fields.get(2), true);
    Assert.assertEquals(struct, struct3);
}

Example 54 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project cdap by caskdata.

the class UnionStructObjectInspector method init.

void init(List<StructObjectInspector> unionObjectInspectors) {
    this.unionObjectInspectors = unionObjectInspectors;
    int totalSize = 0;
    for (StructObjectInspector structObjectInspector : unionObjectInspectors) {
        totalSize += structObjectInspector.getAllStructFieldRefs().size();
    }
    fields = new ArrayList<>(totalSize);
    for (int i = 0; i < unionObjectInspectors.size(); i++) {
        StructObjectInspector oi = unionObjectInspectors.get(i);
        for (StructField sf : oi.getAllStructFieldRefs()) {
            fields.add(new MyField(i, sf));
        }
    }
}

Example 55 with StructField

use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project metacat by Netflix.

the class HiveConnectorInfoConverter method toTableInfo.

/**
 * Converts to TableDto.
 *
 * @param table connector table
 * @return Metacat table Info
 */
@Override
public TableInfo toTableInfo(final QualifiedName name, final Table table) {
    final List<FieldSchema> nonPartitionColumns = (table.getSd() != null) ? table.getSd().getCols() : Collections.emptyList();
    // ignore all exceptions
    try {
        if (nonPartitionColumns.isEmpty()) {
            for (StructField field : HiveTableUtil.getTableStructFields(table)) {
                final FieldSchema fieldSchema = new FieldSchema(field.getFieldName(), field.getFieldObjectInspector().getTypeName(), field.getFieldComment());
                nonPartitionColumns.add(fieldSchema);
            }
        }
    } catch (final Exception e) {
        log.error(e.getMessage(), e);
    }
    final List<FieldSchema> partitionColumns = table.getPartitionKeys();
    final Date creationDate = table.isSetCreateTime() ? epochSecondsToDate(table.getCreateTime()) : null;
    final List<FieldInfo> allFields = Lists.newArrayListWithCapacity(nonPartitionColumns.size() + partitionColumns.size());
    nonPartitionColumns.stream().map(field -> hiveToMetacatField(field, false)).forEachOrdered(allFields::add);
    partitionColumns.stream().map(field -> hiveToMetacatField(field, true)).forEachOrdered(allFields::add);
    final AuditInfo auditInfo = AuditInfo.builder().createdDate(creationDate).build();
    if (null != table.getTableType() && table.getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
        return TableInfo.builder().serde(toStorageInfo(table.getSd(), table.getOwner())).fields(allFields).metadata(table.getParameters()).name(name).auditInfo(auditInfo).view(ViewInfo.builder().viewOriginalText(table.getViewOriginalText()).viewExpandedText(table.getViewExpandedText()).build()).build();
    } else {
        return TableInfo.builder().serde(toStorageInfo(table.getSd(), table.getOwner())).fields(allFields).metadata(table.getParameters()).name(name).auditInfo(auditInfo).build();
    }
}