Examples with LogicalType org.apache.beam.sdk.schemas.Schema.LogicalType used on opensource projects

Example 1 with LogicalType

use of org.apache.beam.sdk.schemas.Schema.LogicalType in project beam by apache.

the class BeamRowToStorageApiProto method fieldDescriptorFromBeamField.

private static FieldDescriptorProto.Builder fieldDescriptorFromBeamField(Field field, int fieldNumber, List<DescriptorProto> nestedTypes) {
    FieldDescriptorProto.Builder fieldDescriptorBuilder = FieldDescriptorProto.newBuilder();
    fieldDescriptorBuilder = fieldDescriptorBuilder.setName(field.getName().toLowerCase());
    fieldDescriptorBuilder = fieldDescriptorBuilder.setNumber(fieldNumber);
    switch(field.getType().getTypeName()) {
        case ROW:
            @Nullable Schema rowSchema = field.getType().getRowSchema();
            if (rowSchema == null) {
                throw new RuntimeException("Unexpected null schema!");
            }
            DescriptorProto nested = descriptorSchemaFromBeamSchema(rowSchema);
            nestedTypes.add(nested);
            fieldDescriptorBuilder = fieldDescriptorBuilder.setType(Type.TYPE_MESSAGE).setTypeName(nested.getName());
            break;
        case ARRAY:
        case ITERABLE:
            @Nullable FieldType elementType = field.getType().getCollectionElementType();
            if (elementType == null) {
                throw new RuntimeException("Unexpected null element type!");
            }
            Preconditions.checkState(!Preconditions.checkNotNull(elementType.getTypeName()).isCollectionType(), "Nested arrays not supported by BigQuery.");
            return fieldDescriptorFromBeamField(Field.of(field.getName(), elementType), fieldNumber, nestedTypes).setLabel(Label.LABEL_REPEATED);
        case LOGICAL_TYPE:
            @Nullable LogicalType<?, ?> logicalType = field.getType().getLogicalType();
            if (logicalType == null) {
                throw new RuntimeException("Unexpected null logical type " + field.getType());
            }
            @Nullable Type type = LOGICAL_TYPES.get(logicalType.getIdentifier());
            if (type == null) {
                throw new RuntimeException("Unsupported logical type " + field.getType());
            }
            fieldDescriptorBuilder = fieldDescriptorBuilder.setType(type);
            break;
        case MAP:
            throw new RuntimeException("Map types not supported by BigQuery.");
        default:
            @Nullable Type primitiveType = PRIMITIVE_TYPES.get(field.getType().getTypeName());
            if (primitiveType == null) {
                throw new RuntimeException("Unsupported type " + field.getType());
            }
            fieldDescriptorBuilder = fieldDescriptorBuilder.setType(primitiveType);
    }
    if (field.getType().getNullable()) {
        fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_OPTIONAL);
    } else {
        fieldDescriptorBuilder = fieldDescriptorBuilder.setLabel(Label.LABEL_REQUIRED);
    }
    return fieldDescriptorBuilder;
}

Example 2 with LogicalType

use of org.apache.beam.sdk.schemas.Schema.LogicalType in project beam by apache.

the class SchemaTranslation method fieldTypeToProto.

private static SchemaApi.FieldType fieldTypeToProto(FieldType fieldType, boolean serializeLogicalType) {
    SchemaApi.FieldType.Builder builder = SchemaApi.FieldType.newBuilder();
    switch(fieldType.getTypeName()) {
        case ROW:
            builder.setRowType(SchemaApi.RowType.newBuilder().setSchema(schemaToProto(fieldType.getRowSchema(), serializeLogicalType)));
            break;
        case ARRAY:
            builder.setArrayType(SchemaApi.ArrayType.newBuilder().setElementType(fieldTypeToProto(fieldType.getCollectionElementType(), serializeLogicalType)));
            break;
        case ITERABLE:
            builder.setIterableType(SchemaApi.IterableType.newBuilder().setElementType(fieldTypeToProto(fieldType.getCollectionElementType(), serializeLogicalType)));
            break;
        case MAP:
            builder.setMapType(SchemaApi.MapType.newBuilder().setKeyType(fieldTypeToProto(fieldType.getMapKeyType(), serializeLogicalType)).setValueType(fieldTypeToProto(fieldType.getMapValueType(), serializeLogicalType)).build());
            break;
        case LOGICAL_TYPE:
            LogicalType logicalType = fieldType.getLogicalType();
            SchemaApi.LogicalType.Builder logicalTypeBuilder;
            if (STANDARD_LOGICAL_TYPES.containsKey(logicalType.getIdentifier())) {
                Preconditions.checkArgument(logicalType.getArgumentType() == null, "Logical type '%s' cannot be used as a logical type, it has a non-null argument type.", logicalType.getIdentifier());
                logicalTypeBuilder = SchemaApi.LogicalType.newBuilder().setRepresentation(fieldTypeToProto(logicalType.getBaseType(), serializeLogicalType)).setUrn(logicalType.getIdentifier());
            } else if (logicalType instanceof UnknownLogicalType) {
                logicalTypeBuilder = SchemaApi.LogicalType.newBuilder().setUrn(logicalType.getIdentifier()).setPayload(ByteString.copyFrom(((UnknownLogicalType) logicalType).getPayload())).setRepresentation(fieldTypeToProto(logicalType.getBaseType(), serializeLogicalType));
                if (logicalType.getArgumentType() != null) {
                    logicalTypeBuilder.setArgumentType(fieldTypeToProto(logicalType.getArgumentType(), serializeLogicalType)).setArgument(fieldValueToProto(logicalType.getArgumentType(), logicalType.getArgument()));
                }
            } else {
                logicalTypeBuilder = SchemaApi.LogicalType.newBuilder().setRepresentation(fieldTypeToProto(logicalType.getBaseType(), serializeLogicalType)).setUrn(URN_BEAM_LOGICAL_JAVASDK);
                if (logicalType.getArgumentType() != null) {
                    logicalTypeBuilder = logicalTypeBuilder.setArgumentType(fieldTypeToProto(logicalType.getArgumentType(), serializeLogicalType)).setArgument(fieldValueToProto(logicalType.getArgumentType(), logicalType.getArgument()));
                }
                if (serializeLogicalType) {
                    logicalTypeBuilder = logicalTypeBuilder.setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(logicalType)));
                }
            }
            builder.setLogicalType(logicalTypeBuilder.build());
            break;
        // but not yet in Java. (BEAM-7554)
        case DATETIME:
            builder.setLogicalType(SchemaApi.LogicalType.newBuilder().setUrn(URN_BEAM_LOGICAL_DATETIME).setRepresentation(fieldTypeToProto(FieldType.INT64, serializeLogicalType)).build());
            break;
        case DECIMAL:
            builder.setLogicalType(SchemaApi.LogicalType.newBuilder().setUrn(URN_BEAM_LOGICAL_DECIMAL).setRepresentation(fieldTypeToProto(FieldType.BYTES, serializeLogicalType)).build());
            break;
        case BYTE:
            builder.setAtomicType(SchemaApi.AtomicType.BYTE);
            break;
        case INT16:
            builder.setAtomicType(SchemaApi.AtomicType.INT16);
            break;
        case INT32:
            builder.setAtomicType(SchemaApi.AtomicType.INT32);
            break;
        case INT64:
            builder.setAtomicType(SchemaApi.AtomicType.INT64);
            break;
        case FLOAT:
            builder.setAtomicType(SchemaApi.AtomicType.FLOAT);
            break;
        case DOUBLE:
            builder.setAtomicType(SchemaApi.AtomicType.DOUBLE);
            break;
        case STRING:
            builder.setAtomicType(SchemaApi.AtomicType.STRING);
            break;
        case BOOLEAN:
            builder.setAtomicType(SchemaApi.AtomicType.BOOLEAN);
            break;
        case BYTES:
            builder.setAtomicType(SchemaApi.AtomicType.BYTES);
            break;
    }
    builder.setNullable(fieldType.getNullable());
    return builder.build();
}