Search in sources :

Example 6 with NamedTypeSignature

use of io.prestosql.spi.type.NamedTypeSignature in project hetu-core by openlookeng.

the class ParquetColumnIOConverter method constructField.

public static Optional<Field> constructField(Type type, ColumnIO columnIO) {
    if (columnIO == null) {
        return Optional.empty();
    }
    boolean required = columnIO.getType().getRepetition() != OPTIONAL;
    int repetitionLevel = columnRepetitionLevel(columnIO);
    int definitionLevel = columnDefinitionLevel(columnIO);
    if (ROW.equals(type.getTypeSignature().getBase())) {
        GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
        List<Type> parameters = type.getTypeParameters();
        ImmutableList.Builder<Optional<Field>> fieldsBuilder = ImmutableList.builder();
        List<TypeSignatureParameter> fields = type.getTypeSignature().getParameters();
        boolean structHasParameters = false;
        for (int i = 0; i < fields.size(); i++) {
            NamedTypeSignature namedTypeSignature = fields.get(i).getNamedTypeSignature();
            String name = namedTypeSignature.getName().get().toLowerCase(Locale.ENGLISH);
            Optional<Field> field = constructField(parameters.get(i), lookupColumnByName(groupColumnIO, name));
            structHasParameters |= field.isPresent();
            fieldsBuilder.add(field);
        }
        if (structHasParameters) {
            return Optional.of(new GroupField(type, repetitionLevel, definitionLevel, required, fieldsBuilder.build()));
        }
        return Optional.empty();
    }
    if (MAP.equals(type.getTypeSignature().getBase())) {
        GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
        MapType mapType = (MapType) type;
        GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO);
        if (keyValueColumnIO.getChildrenCount() != 2) {
            return Optional.empty();
        }
        Optional<Field> keyField = constructField(mapType.getKeyType(), keyValueColumnIO.getChild(0));
        Optional<Field> valueField = constructField(mapType.getValueType(), keyValueColumnIO.getChild(1));
        return Optional.of(new GroupField(type, repetitionLevel, definitionLevel, required, ImmutableList.of(keyField, valueField)));
    }
    if (ARRAY.equals(type.getTypeSignature().getBase())) {
        GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
        List<Type> types = type.getTypeParameters();
        if (groupColumnIO.getChildrenCount() != 1) {
            return Optional.empty();
        }
        Optional<Field> field = constructField(types.get(0), getArrayElementColumn(groupColumnIO.getChild(0)));
        return Optional.of(new GroupField(type, repetitionLevel, definitionLevel, required, ImmutableList.of(field)));
    }
    PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO;
    RichColumnDescriptor column = new RichColumnDescriptor(primitiveColumnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType());
    return Optional.of(new PrimitiveField(type, repetitionLevel, definitionLevel, required, column, primitiveColumnIO.getId()));
}
Also used : Optional(java.util.Optional) ImmutableList(com.google.common.collect.ImmutableList) GroupField(io.prestosql.parquet.GroupField) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) MapType(io.prestosql.spi.type.MapType) PrimitiveColumnIO(org.apache.parquet.io.PrimitiveColumnIO) GroupField(io.prestosql.parquet.GroupField) PrimitiveField(io.prestosql.parquet.PrimitiveField) Field(io.prestosql.parquet.Field) MapType(io.prestosql.spi.type.MapType) Type(io.prestosql.spi.type.Type) GroupColumnIO(org.apache.parquet.io.GroupColumnIO) TypeSignatureParameter(io.prestosql.spi.type.TypeSignatureParameter) PrimitiveField(io.prestosql.parquet.PrimitiveField)

Example 7 with NamedTypeSignature

use of io.prestosql.spi.type.NamedTypeSignature in project hetu-core by openlookeng.

the class TestRowParametricType method testTypeSignatureRoundTrip.

@Test
public void testTypeSignatureRoundTrip() {
    TypeManager typeManager = new InternalTypeManager(createTestMetadataManager().getFunctionAndTypeManager());
    TypeSignature typeSignature = new TypeSignature(ROW, TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("col1", false)), new TypeSignature(BIGINT))), TypeSignatureParameter.of(new NamedTypeSignature(Optional.of(new RowFieldName("col2", true)), new TypeSignature(DOUBLE))));
    List<TypeParameter> parameters = typeSignature.getParameters().stream().map(parameter -> TypeParameter.of(parameter, typeManager)).collect(Collectors.toList());
    Type rowType = RowParametricType.ROW.createType(typeManager, parameters);
    assertEquals(rowType.getTypeSignature(), typeSignature);
}
Also used : DOUBLE(io.prestosql.spi.type.StandardTypes.DOUBLE) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) TypeManager(io.prestosql.spi.type.TypeManager) Assert.assertEquals(org.testng.Assert.assertEquals) TypeSignatureParameter(io.prestosql.spi.type.TypeSignatureParameter) Test(org.testng.annotations.Test) Collectors(java.util.stream.Collectors) RowParametricType(io.prestosql.spi.type.RowParametricType) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) List(java.util.List) TypeParameter(io.prestosql.spi.type.TypeParameter) Optional(java.util.Optional) RowFieldName(io.prestosql.spi.type.RowFieldName) Type(io.prestosql.spi.type.Type) TypeSignature(io.prestosql.spi.type.TypeSignature) ROW(io.prestosql.spi.type.StandardTypes.ROW) BIGINT(io.prestosql.spi.type.StandardTypes.BIGINT) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) TypeSignature(io.prestosql.spi.type.TypeSignature) TypeParameter(io.prestosql.spi.type.TypeParameter) RowParametricType(io.prestosql.spi.type.RowParametricType) Type(io.prestosql.spi.type.Type) RowFieldName(io.prestosql.spi.type.RowFieldName) TypeManager(io.prestosql.spi.type.TypeManager) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) Test(org.testng.annotations.Test)

Example 8 with NamedTypeSignature

use of io.prestosql.spi.type.NamedTypeSignature in project boostkit-bigdata by kunpengcompute.

the class HiveTypeTranslator method translate.

@Override
public TypeInfo translate(Type type) {
    if (BOOLEAN.equals(type)) {
        return HIVE_BOOLEAN.getTypeInfo();
    }
    if (BIGINT.equals(type)) {
        return HIVE_LONG.getTypeInfo();
    }
    if (INTEGER.equals(type)) {
        return HIVE_INT.getTypeInfo();
    }
    if (SMALLINT.equals(type)) {
        return HIVE_SHORT.getTypeInfo();
    }
    if (TINYINT.equals(type)) {
        return HIVE_BYTE.getTypeInfo();
    }
    if (REAL.equals(type)) {
        return HIVE_FLOAT.getTypeInfo();
    }
    if (DOUBLE.equals(type)) {
        return HIVE_DOUBLE.getTypeInfo();
    }
    if (type instanceof VarcharType) {
        VarcharType varcharType = (VarcharType) type;
        if (varcharType.isUnbounded()) {
            return HIVE_STRING.getTypeInfo();
        }
        if (varcharType.getBoundedLength() <= HiveVarchar.MAX_VARCHAR_LENGTH) {
            return getVarcharTypeInfo(varcharType.getBoundedLength());
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported VARCHAR types: VARCHAR(<=%d), VARCHAR.", type, HiveVarchar.MAX_VARCHAR_LENGTH));
    }
    if (type instanceof CharType) {
        CharType charType = (CharType) type;
        int charLength = charType.getLength();
        if (charLength <= HiveChar.MAX_CHAR_LENGTH) {
            return getCharTypeInfo(charLength);
        }
        throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s. Supported CHAR types: CHAR(<=%d).", type, HiveChar.MAX_CHAR_LENGTH));
    }
    if (VARBINARY.equals(type)) {
        return HIVE_BINARY.getTypeInfo();
    }
    if (DATE.equals(type)) {
        return HIVE_DATE.getTypeInfo();
    }
    if (TIMESTAMP.equals(type)) {
        return HIVE_TIMESTAMP.getTypeInfo();
    }
    if (type instanceof DecimalType) {
        DecimalType decimalType = (DecimalType) type;
        return new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
    }
    if (isArrayType(type)) {
        TypeInfo elementType = translate(type.getTypeParameters().get(0));
        return getListTypeInfo(elementType);
    }
    if (isMapType(type)) {
        TypeInfo keyType = translate(type.getTypeParameters().get(0));
        TypeInfo valueType = translate(type.getTypeParameters().get(1));
        return getMapTypeInfo(keyType, valueType);
    }
    if (isRowType(type)) {
        ImmutableList.Builder<String> fieldNames = ImmutableList.builder();
        for (TypeSignatureParameter parameter : type.getTypeSignature().getParameters()) {
            if (!parameter.isNamedTypeSignature()) {
                throw new IllegalArgumentException(format("Expected all parameters to be named type, but got %s", parameter));
            }
            NamedTypeSignature namedTypeSignature = parameter.getNamedTypeSignature();
            if (!namedTypeSignature.getName().isPresent()) {
                throw new PrestoException(NOT_SUPPORTED, format("Anonymous row type is not supported in Hive. Please give each field a name: %s", type));
            }
            fieldNames.add(namedTypeSignature.getName().get());
        }
        return getStructTypeInfo(fieldNames.build(), type.getTypeParameters().stream().map(this::translate).collect(toList()));
    }
    throw new PrestoException(NOT_SUPPORTED, format("Unsupported Hive type: %s", type));
}
Also used : VarcharType(io.prestosql.spi.type.VarcharType) ImmutableList(com.google.common.collect.ImmutableList) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) PrestoException(io.prestosql.spi.PrestoException) TypeInfoFactory.getCharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getCharTypeInfo) TypeInfoFactory.getStructTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getStructTypeInfo) TypeInfoFactory.getVarcharTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getVarcharTypeInfo) TypeInfoFactory.getMapTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getMapTypeInfo) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) TypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfo) TypeInfoFactory.getListTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory.getListTypeInfo) DecimalTypeInfo(org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo) TypeSignatureParameter(io.prestosql.spi.type.TypeSignatureParameter) DecimalType(io.prestosql.spi.type.DecimalType) CharType(io.prestosql.spi.type.CharType)

Example 9 with NamedTypeSignature

use of io.prestosql.spi.type.NamedTypeSignature in project boostkit-bigdata by kunpengcompute.

the class ParquetColumnIOConverter method constructField.

public static Optional<Field> constructField(Type type, ColumnIO columnIO) {
    if (columnIO == null) {
        return Optional.empty();
    }
    boolean required = columnIO.getType().getRepetition() != OPTIONAL;
    int repetitionLevel = columnRepetitionLevel(columnIO);
    int definitionLevel = columnDefinitionLevel(columnIO);
    if (ROW.equals(type.getTypeSignature().getBase())) {
        GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
        List<Type> parameters = type.getTypeParameters();
        ImmutableList.Builder<Optional<Field>> fieldsBuilder = ImmutableList.builder();
        List<TypeSignatureParameter> fields = type.getTypeSignature().getParameters();
        boolean structHasParameters = false;
        for (int i = 0; i < fields.size(); i++) {
            NamedTypeSignature namedTypeSignature = fields.get(i).getNamedTypeSignature();
            String name = namedTypeSignature.getName().get().toLowerCase(Locale.ENGLISH);
            Optional<Field> field = constructField(parameters.get(i), lookupColumnByName(groupColumnIO, name));
            structHasParameters |= field.isPresent();
            fieldsBuilder.add(field);
        }
        if (structHasParameters) {
            return Optional.of(new GroupField(type, repetitionLevel, definitionLevel, required, fieldsBuilder.build()));
        }
        return Optional.empty();
    }
    if (MAP.equals(type.getTypeSignature().getBase())) {
        GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
        MapType mapType = (MapType) type;
        GroupColumnIO keyValueColumnIO = getMapKeyValueColumn(groupColumnIO);
        if (keyValueColumnIO.getChildrenCount() != 2) {
            return Optional.empty();
        }
        Optional<Field> keyField = constructField(mapType.getKeyType(), keyValueColumnIO.getChild(0));
        Optional<Field> valueField = constructField(mapType.getValueType(), keyValueColumnIO.getChild(1));
        return Optional.of(new GroupField(type, repetitionLevel, definitionLevel, required, ImmutableList.of(keyField, valueField)));
    }
    if (ARRAY.equals(type.getTypeSignature().getBase())) {
        GroupColumnIO groupColumnIO = (GroupColumnIO) columnIO;
        List<Type> types = type.getTypeParameters();
        if (groupColumnIO.getChildrenCount() != 1) {
            return Optional.empty();
        }
        Optional<Field> field = constructField(types.get(0), getArrayElementColumn(groupColumnIO.getChild(0)));
        return Optional.of(new GroupField(type, repetitionLevel, definitionLevel, required, ImmutableList.of(field)));
    }
    PrimitiveColumnIO primitiveColumnIO = (PrimitiveColumnIO) columnIO;
    RichColumnDescriptor column = new RichColumnDescriptor(primitiveColumnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType());
    return Optional.of(new PrimitiveField(type, repetitionLevel, definitionLevel, required, column, primitiveColumnIO.getId()));
}
Also used : Optional(java.util.Optional) ImmutableList(com.google.common.collect.ImmutableList) GroupField(io.prestosql.parquet.GroupField) RichColumnDescriptor(io.prestosql.parquet.RichColumnDescriptor) NamedTypeSignature(io.prestosql.spi.type.NamedTypeSignature) MapType(io.prestosql.spi.type.MapType) PrimitiveColumnIO(org.apache.parquet.io.PrimitiveColumnIO) GroupField(io.prestosql.parquet.GroupField) PrimitiveField(io.prestosql.parquet.PrimitiveField) Field(io.prestosql.parquet.Field) MapType(io.prestosql.spi.type.MapType) Type(io.prestosql.spi.type.Type) GroupColumnIO(org.apache.parquet.io.GroupColumnIO) TypeSignatureParameter(io.prestosql.spi.type.TypeSignatureParameter) PrimitiveField(io.prestosql.parquet.PrimitiveField)

Aggregations

NamedTypeSignature (io.prestosql.spi.type.NamedTypeSignature)9 TypeSignatureParameter (io.prestosql.spi.type.TypeSignatureParameter)9 ImmutableList (com.google.common.collect.ImmutableList)8 Type (io.prestosql.spi.type.Type)7 PrestoException (io.prestosql.spi.PrestoException)5 RowFieldName (io.prestosql.spi.type.RowFieldName)5 TypeSignature (io.prestosql.spi.type.TypeSignature)4 VarcharType (io.prestosql.spi.type.VarcharType)4 Optional (java.util.Optional)4 DecimalTypeInfo (org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo)4 TypeInfo (org.apache.hadoop.hive.serde2.typeinfo.TypeInfo)4 CharType (io.prestosql.spi.type.CharType)3 DecimalType (io.prestosql.spi.type.DecimalType)3 VarcharType.createUnboundedVarcharType (io.prestosql.spi.type.VarcharType.createUnboundedVarcharType)3 Field (io.prestosql.parquet.Field)2 GroupField (io.prestosql.parquet.GroupField)2 PrimitiveField (io.prestosql.parquet.PrimitiveField)2 RichColumnDescriptor (io.prestosql.parquet.RichColumnDescriptor)2 MapType (io.prestosql.spi.type.MapType)2 TypeManager (io.prestosql.spi.type.TypeManager)2