Examples with Type org.apache.parquet.schema.Type used on opensource projects

Example 31 with Type

use of org.apache.parquet.schema.Type in project presto by prestodb.

the class TestDataWritableWriter method writeGroupFields.

/**
 * It writes all the fields contained inside a group to the RecordConsumer.
 *
 * @param value The list of values contained in the group.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the group schema.
 */
private void writeGroupFields(final Object value, final StructObjectInspector inspector, final GroupType type) {
    if (value != null) {
        List<? extends StructField> fields = inspector.getAllStructFieldRefs();
        List<Object> fieldValuesList = inspector.getStructFieldsDataAsList(value);
        for (int i = 0; i < type.getFieldCount(); i++) {
            Type fieldType = type.getType(i);
            String fieldName = fieldType.getName();
            Object fieldValue = fieldValuesList.get(i);
            if (fieldValue != null) {
                ObjectInspector fieldInspector = fields.get(i).getFieldObjectInspector();
                recordConsumer.startField(fieldName, i);
                writeValue(fieldValue, fieldInspector, fieldType);
                recordConsumer.endField(fieldName, i);
            }
        }
    }
}

Example 32 with Type

use of org.apache.parquet.schema.Type in project presto by prestodb.

the class TestDataWritableWriter method writeArray.

/**
 * It writes a list type and its array elements to the Parquet RecordConsumer.
 * This is called when the original type (LIST) is detected by writeValue()/
 * This function assumes the following schema:
 * optional group arrayCol (LIST) {
 * repeated group array {
 * optional TYPE array_element;
 * }
 * }
 *
 * @param value The object that contains the array values.
 * @param inspector The object inspector used to get the correct value type.
 * @param type Type that contains information about the group (LIST) schema.
 */
private void writeArray(final Object value, final ListObjectInspector inspector, final GroupType type) {
    // Get the internal array structure
    GroupType repeatedType = type.getType(0).asGroupType();
    recordConsumer.startGroup();
    List<?> arrayValues = inspector.getList(value);
    if (!arrayValues.isEmpty()) {
        recordConsumer.startField(repeatedType.getName(), 0);
        ObjectInspector elementInspector = inspector.getListElementObjectInspector();
        Type elementType = repeatedType.getType(0);
        String elementName = elementType.getName();
        for (Object element : arrayValues) {
            recordConsumer.startGroup();
            if (element != null) {
                recordConsumer.startField(elementName, 0);
                writeValue(element, elementInspector, elementType);
                recordConsumer.endField(elementName, 0);
            }
            recordConsumer.endGroup();
        }
        recordConsumer.endField(repeatedType.getName(), 0);
    }
    recordConsumer.endGroup();
}

Example 33 with Type

use of org.apache.parquet.schema.Type in project presto by prestodb.

the class TestDataWritableWriter method writeSingleLevelArray.

private void writeSingleLevelArray(final Object value, final ListObjectInspector inspector, final GroupType type) {
    // Get the internal array structure
    Type elementType = type.getType(0);
    recordConsumer.startGroup();
    List<?> arrayValues = inspector.getList(value);
    if (!arrayValues.isEmpty()) {
        recordConsumer.startField(elementType.getName(), 0);
        ObjectInspector elementInspector = inspector.getListElementObjectInspector();
        for (Object element : arrayValues) {
            if (element == null) {
                throw new IllegalArgumentException("Array elements are requires in given schema definition");
            }
            writeValue(element, elementInspector, elementType);
        }
        recordConsumer.endField(elementType.getName(), 0);
    }
    recordConsumer.endGroup();
}

Example 34 with Type

use of org.apache.parquet.schema.Type in project presto by prestodb.

the class SingleLevelArrayMapKeyValuesSchemaConverter method convertMapType.

// An optional group containing a repeated anonymous group "map", containing
// 2 elements: "key", "value"
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo, final Repetition repetition) {
    final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED);
    final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo());
    return mapType(repetition, name, "map", keyType, valueType);
}

Example 35 with Type

use of org.apache.parquet.schema.Type in project presto by prestodb.

the class MapKeyValuesSchemaConverter method convertMapType.

// An optional group containing a repeated anonymous group "map", containing
// 2 elements: "key", "value"
private static GroupType convertMapType(final String name, final MapTypeInfo typeInfo) {
    final Type keyType = convertType(ParquetHiveSerDe.MAP_KEY.toString(), typeInfo.getMapKeyTypeInfo(), Repetition.REQUIRED);
    final Type valueType = convertType(ParquetHiveSerDe.MAP_VALUE.toString(), typeInfo.getMapValueTypeInfo());
    return mapType(Repetition.OPTIONAL, name, "map", keyType, valueType);
}