Examples with LegacySQLTypeName com.google.cloud.bigquery.LegacySQLTypeName used on opensource projects

Example 1 with LegacySQLTypeName

use of com.google.cloud.bigquery.LegacySQLTypeName in project DataflowTemplates by GoogleCloudPlatform.

the class SchemaUtils method convertProtoFieldDescriptorToBigQueryField.

/**
 * Handlers proto field to BigQuery field conversion.
 */
private static TableFieldSchema convertProtoFieldDescriptorToBigQueryField(FieldDescriptor fieldDescriptor, boolean preserveProtoFieldNames, @Nullable FieldDescriptor parent, int nestingLevel) {
    TableFieldSchema schema = new TableFieldSchema();
    String jsonName = fieldDescriptor.getJsonName();
    schema.setName(preserveProtoFieldNames || Strings.isNullOrEmpty(jsonName) ? fieldDescriptor.getName() : jsonName);
    LegacySQLTypeName sqlType = convertProtoTypeToSqlType(fieldDescriptor.getJavaType());
    schema.setType(sqlType.toString());
    if (sqlType == LegacySQLTypeName.RECORD) {
        if (nestingLevel > MAX_BIG_QUERY_RECORD_NESTING) {
            throw new IllegalArgumentException(String.format("Record field `%s.%s` is at BigQuery's nesting limit of %s, but it contains" + " message field `%s` of type `%s`. This could be caused by circular message" + " references, including a self-referential message.", parent.getMessageType().getName(), parent.getName(), MAX_BIG_QUERY_RECORD_NESTING, fieldDescriptor.getName(), fieldDescriptor.getMessageType().getName()));
        }
        List<TableFieldSchema> subFields = fieldDescriptor.getMessageType().getFields().stream().map(fd -> convertProtoFieldDescriptorToBigQueryField(fd, preserveProtoFieldNames, fieldDescriptor, nestingLevel + 1)).collect(toList());
        schema.setFields(subFields);
    }
    if (fieldDescriptor.isRepeated()) {
        schema.setMode("REPEATED");
    } else if (fieldDescriptor.isRequired()) {
        schema.setMode("REQUIRED");
    } else {
        schema.setMode("NULLABLE");
    }
    return schema;
}

Example 2 with LegacySQLTypeName

use of com.google.cloud.bigquery.LegacySQLTypeName in project DataflowTemplates by GoogleCloudPlatform.

the class BigQueryTableRowCleaner method cleanTableRowField.

/**
 * Cleans the TableRow data for a given rowKey if the value does not match requirements of the
 * BigQuery Table column type.
 *
 * @param row a TableRow object to clean.
 * @param tableFields a FieldList of Bigquery columns.
 * @param rowKey a String with the name of the field to clean.
 */
public static void cleanTableRowField(TableRow row, FieldList tableFields, String rowKey) {
    Field tableField = tableFields.get(rowKey);
    LegacySQLTypeName fieldType = tableField.getType();
    if (fieldType == LegacySQLTypeName.STRING) {
        cleanTableRowFieldStrings(row, tableFields, rowKey);
    } else if (fieldType == LegacySQLTypeName.DATE) {
        cleanTableRowFieldDates(row, tableFields, rowKey);
    }
}

Example 3 with LegacySQLTypeName

use of com.google.cloud.bigquery.LegacySQLTypeName in project workbench by all-of-us.

the class BigQueryBaseTest method parseSchema.

private Schema parseSchema(Column[] columns) {
    List<Field> schemaFields = new ArrayList<>();
    for (Column column : columns) {
        String typeString = column.getType();
        LegacySQLTypeName fieldType;
        switch(column.getType().toLowerCase()) {
            case "string":
                fieldType = LegacySQLTypeName.STRING;
                break;
            case "integer":
                fieldType = LegacySQLTypeName.INTEGER;
                break;
            case "timestamp":
                fieldType = LegacySQLTypeName.TIMESTAMP;
                break;
            case "float":
                fieldType = LegacySQLTypeName.FLOAT;
                break;
            case "boolean":
                fieldType = LegacySQLTypeName.BOOLEAN;
                break;
            case "date":
                fieldType = LegacySQLTypeName.DATE;
                break;
            default:
                throw new IllegalArgumentException("Unrecognized field type '" + typeString + "'.");
        }
        schemaFields.add(Field.of(column.getName(), fieldType));
    }
    return Schema.of(schemaFields);
}

Example 4 with LegacySQLTypeName

use of com.google.cloud.bigquery.LegacySQLTypeName in project DataflowTemplates by GoogleCloudPlatform.

the class SchemaUtils method createBigQuerySchema.

/**
 * Infers the proper {@link TableSchema} for a Protobuf message.
 *
 * <p>The name value for each field in the returned schema will default to {@link
 * FieldDescriptor#getJsonName()}. If this value is not present, then it will use {@link
 * FieldDescriptor#getName()}.
 *
 * <p>Type mappings follow the rules:
 *
 * <ul>
 *   <li>Integers and longs map to {@link LegacySQLTypeName#INTEGER}.
 *   <li>Floats and doubles map to {@link LegacySQLTypeName#FLOAT}.
 *   <li>Booleans map to {@link LegacySQLTypeName#BOOLEAN}.
 *   <li>Strings and enums map to {@link LegacySQLTypeName#STRING}.
 *   <li>Byte strings map to {@link LegacySQLTypeName#BYTES}.
 *   <li>Messages and maps map to {@link LegacySQLTypeName#RECORD}.
 * </ul>
 *
 * <p>Fields marked as `oneof` in the proto definition will be expanded out into individual
 * fields. `oneof` fields are incompatible with BigQuery otherwise.
 *
 * <p>An error will be thrown if excessive RECORD nesting is detected. BigQuery has more
 * restrictive RECORD nesting limits than Protobuf has message nesting. Circular message
 * references and self-referential messages are not supported for this reason.
 *
 * <p>Proto repeated fields will be marked as REPEATED in BigQuery. Required and optional fields
 * will be marked as REQUIRED and NULLABLE respectively.
 *
 * <p>No description or policy tags will be set for any of the fields.
 *
 * @param descriptor a proto {@link Descriptor} to be converted into a BigQuery schema
 * @param preserveProtoFieldNames true to keep proto snake_case. False to use lowerCamelCase. If
 *     set to false and {@link FieldDescriptor#getJsonName()} is not set, then snake_case will be
 *     used.
 * @return a full BigQuery schema definition
 */
public static TableSchema createBigQuerySchema(Descriptor descriptor, boolean preserveProtoFieldNames) {
    // TableSchema and TableFieldSchema work better with Beam than Schema and Field.
    List<TableFieldSchema> fields = descriptor.getFields().stream().map(fd -> convertProtoFieldDescriptorToBigQueryField(fd, preserveProtoFieldNames, /* parent= */
    null, /* nestingLevel= */
    1)).collect(toList());
    TableSchema schema = new TableSchema();
    schema.setFields(fields);
    return schema;
}

Example 5 with LegacySQLTypeName

use of com.google.cloud.bigquery.LegacySQLTypeName in project java-retail by googleapis.

the class SetupCleanup method getGson.

public static Gson getGson() {
    JsonDeserializer<LegacySQLTypeName> typeDeserializer = (jsonElement, type, deserializationContext) -> {
        return LegacySQLTypeName.valueOf(jsonElement.getAsString());
    };
    JsonDeserializer<FieldList> subFieldsDeserializer = (jsonElement, type, deserializationContext) -> {
        Field[] fields = deserializationContext.deserialize(jsonElement.getAsJsonArray(), Field[].class);
        return FieldList.of(fields);
    };
    return new GsonBuilder().registerTypeAdapter(LegacySQLTypeName.class, typeDeserializer).registerTypeAdapter(FieldList.class, subFieldsDeserializer).create();
}