Examples with FieldType org.apache.beam.sdk.schemas.Schema.FieldType used on opensource projects

Example 36 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class ThriftSchema method beamType.

private <EnumT extends Enum<EnumT> & TEnum> FieldType beamType(FieldValueMetaData metadata) {
    if (metadata.isTypedef()) {
        final FieldType beamType = typedefs.get(metadata.getTypedefName());
        if (beamType != null) {
            return beamType;
        }
    }
    switch(metadata.type) {
        case TType.BOOL:
            return FieldType.BOOLEAN;
        case TType.BYTE:
            return FieldType.BYTE;
        case TType.I16:
            return FieldType.INT16;
        case TType.I32:
            return FieldType.INT32;
        case TType.I64:
            return FieldType.INT64;
        case TType.DOUBLE:
            return FieldType.DOUBLE;
        case TType.STRING:
            return metadata.isBinary() ? FieldType.BYTES : FieldType.STRING;
        case TType.SET:
            final FieldValueMetaData setElemMetadata = ((SetMetaData) metadata).elemMetaData;
            return FieldType.iterable(beamType(setElemMetadata));
        case TType.LIST:
            final FieldValueMetaData listElemMetadata = ((ListMetaData) metadata).elemMetaData;
            return FieldType.array(beamType(listElemMetadata));
        case TType.MAP:
            final MapMetaData mapMetadata = ((MapMetaData) metadata);
            return FieldType.map(beamType(mapMetadata.keyMetaData), beamType(mapMetadata.valueMetaData));
        case TType.STRUCT:
            final StructMetaData structMetadata = ((StructMetaData) metadata);
            return FieldType.row(schemaFor(structMetadata.structClass));
        case TType.ENUM:
            @SuppressWarnings("unchecked") final Class<EnumT> enumClass = (Class<EnumT>) ((EnumMetaData) metadata).enumClass;
            // it's either "nullness" or "unsafe", apparently
            @SuppressWarnings("nullness") final EnumT @NonNull [] enumConstants = enumClass.getEnumConstants();
            final String[] enumValues = Stream.of(enumConstants).map(EnumT::name).toArray(String[]::new);
            return FieldType.logicalType(EnumerationType.create(enumValues));
        default:
            throw new IllegalArgumentException("Unsupported thrift type code: " + metadata.type);
    }
}

Example 37 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class ProtoSchemaTranslator method getOptions.

private static Schema.Options.Builder getOptions(String prefix, Map<FieldDescriptor, Object> allFields) {
    Schema.Options.Builder optionsBuilder = Schema.Options.builder();
    for (Map.Entry<FieldDescriptor, Object> entry : allFields.entrySet()) {
        FieldDescriptor fieldDescriptor = entry.getKey();
        FieldType fieldType = beamFieldTypeFromProtoField(fieldDescriptor);
        switch(fieldType.getTypeName()) {
            case BYTE:
            case BYTES:
            case INT16:
            case INT32:
            case INT64:
            case DECIMAL:
            case FLOAT:
            case DOUBLE:
            case STRING:
            case BOOLEAN:
            case LOGICAL_TYPE:
            case ROW:
            case ARRAY:
            case ITERABLE:
                Field field = Field.of("OPTION", fieldType);
                ProtoDynamicMessageSchema schema = ProtoDynamicMessageSchema.forSchema(Schema.of(field));
                optionsBuilder.setOption(prefix + fieldDescriptor.getFullName(), fieldType, schema.createConverter(field).convertFromProtoValue(entry.getValue()));
                break;
            case MAP:
            case DATETIME:
            default:
                throw new IllegalStateException("These datatypes are not possible in extentions.");
        }
    }
    return optionsBuilder;
}

Example 38 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class ProtoSchemaTranslator method beamFieldTypeFromSingularProtoField.

private static FieldType beamFieldTypeFromSingularProtoField(Descriptors.FieldDescriptor protoFieldDescriptor) {
    Descriptors.FieldDescriptor.Type fieldDescriptor = protoFieldDescriptor.getType();
    FieldType fieldType;
    switch(fieldDescriptor) {
        case INT32:
            fieldType = FieldType.INT32;
            break;
        case INT64:
            fieldType = FieldType.INT64;
            break;
        case FLOAT:
            fieldType = FieldType.FLOAT;
            break;
        case DOUBLE:
            fieldType = FieldType.DOUBLE;
            break;
        case BOOL:
            fieldType = FieldType.BOOLEAN;
            break;
        case STRING:
            fieldType = FieldType.STRING;
            break;
        case BYTES:
            fieldType = FieldType.BYTES;
            break;
        case UINT32:
            fieldType = FieldType.logicalType(new UInt32());
            break;
        case SINT32:
            fieldType = FieldType.logicalType(new SInt32());
            break;
        case FIXED32:
            fieldType = FieldType.logicalType(new Fixed32());
            break;
        case SFIXED32:
            fieldType = FieldType.logicalType(new SFixed32());
            break;
        case UINT64:
            fieldType = FieldType.logicalType(new UInt64());
            break;
        case SINT64:
            fieldType = FieldType.logicalType(new SInt64());
            break;
        case FIXED64:
            fieldType = FieldType.logicalType(new Fixed64());
            break;
        case SFIXED64:
            fieldType = FieldType.logicalType(new SFixed64());
            break;
        case ENUM:
            Map<String, Integer> enumValues = Maps.newHashMap();
            for (EnumValueDescriptor enumValue : protoFieldDescriptor.getEnumType().getValues()) {
                if (enumValues.putIfAbsent(enumValue.getName(), enumValue.getNumber()) != null) {
                    throw new RuntimeException("Aliased enumerations not currently supported.");
                }
            }
            fieldType = FieldType.logicalType(EnumerationType.create(enumValues));
            break;
        case MESSAGE:
        case GROUP:
            String fullName = protoFieldDescriptor.getMessageType().getFullName();
            switch(fullName) {
                case "google.protobuf.Timestamp":
                    fieldType = FieldType.logicalType(new NanosInstant());
                    break;
                case "google.protobuf.Int32Value":
                case "google.protobuf.UInt32Value":
                case "google.protobuf.Int64Value":
                case "google.protobuf.UInt64Value":
                case "google.protobuf.FloatValue":
                case "google.protobuf.DoubleValue":
                case "google.protobuf.StringValue":
                case "google.protobuf.BoolValue":
                case "google.protobuf.BytesValue":
                    fieldType = beamFieldTypeFromSingularProtoField(protoFieldDescriptor.getMessageType().findFieldByNumber(1));
                    break;
                case "google.protobuf.Duration":
                    fieldType = FieldType.logicalType(new NanosDuration());
                    break;
                case "google.protobuf.Any":
                    throw new RuntimeException("Any not yet supported");
                default:
                    fieldType = FieldType.row(getSchema(protoFieldDescriptor.getMessageType()));
            }
            // all messages are nullable in Proto
            if (protoFieldDescriptor.isOptional()) {
                fieldType = fieldType.withNullable(true);
            }
            break;
        default:
            throw new RuntimeException("Field type not matched.");
    }
    return fieldType;
}

Example 39 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class ProtoSchemaTranslator method beamFieldTypeFromProtoField.

private static FieldType beamFieldTypeFromProtoField(Descriptors.FieldDescriptor protoFieldDescriptor) {
    FieldType fieldType = null;
    if (protoFieldDescriptor.isMapField()) {
        FieldDescriptor keyFieldDescriptor = protoFieldDescriptor.getMessageType().findFieldByName("key");
        FieldDescriptor valueFieldDescriptor = protoFieldDescriptor.getMessageType().findFieldByName("value");
        fieldType = FieldType.map(beamFieldTypeFromProtoField(keyFieldDescriptor).withNullable(false), beamFieldTypeFromProtoField(valueFieldDescriptor).withNullable(false));
    } else if (protoFieldDescriptor.isRepeated()) {
        fieldType = FieldType.array(beamFieldTypeFromSingularProtoField(protoFieldDescriptor).withNullable(false));
    } else {
        fieldType = beamFieldTypeFromSingularProtoField(protoFieldDescriptor);
    }
    return fieldType;
}

Example 40 with FieldType

use of org.apache.beam.sdk.schemas.Schema.FieldType in project beam by apache.

the class BigQueryUtils method convertAvroArray.

private static Object convertAvroArray(FieldType beamField, Object value, BigQueryUtils.ConversionOptions options) {
    // Check whether the type of array element is equal.
    List<Object> values = (List<Object>) value;
    List<Object> ret = new ArrayList<>();
    FieldType collectionElement = beamField.getCollectionElementType();
    for (Object v : values) {
        ret.add(convertAvroFormat(collectionElement, v, options));
    }
    return ret;
}