Examples with TableFieldSchema com.google.api.services.bigquery.model.TableFieldSchema used on opensource projects

Example 51 with TableFieldSchema

use of com.google.api.services.bigquery.model.TableFieldSchema in project beam by apache.

the class BigQueryAvroUtils method convertField.

private static Field convertField(TableFieldSchema bigQueryField) {
    ImmutableCollection<Type> avroTypes = BIG_QUERY_TO_AVRO_TYPES.get(bigQueryField.getType());
    if (avroTypes.isEmpty()) {
        throw new IllegalArgumentException("Unable to map BigQuery field type " + bigQueryField.getType() + " to avro type.");
    }
    Type avroType = avroTypes.iterator().next();
    Schema elementSchema;
    if (avroType == Type.RECORD) {
        elementSchema = toGenericAvroSchema(bigQueryField.getName(), bigQueryField.getFields());
    } else {
        elementSchema = Schema.create(avroType);
    }
    Schema fieldSchema;
    if (bigQueryField.getMode() == null || "NULLABLE".equals(bigQueryField.getMode())) {
        fieldSchema = Schema.createUnion(Schema.create(Type.NULL), elementSchema);
    } else if ("REQUIRED".equals(bigQueryField.getMode())) {
        fieldSchema = elementSchema;
    } else if ("REPEATED".equals(bigQueryField.getMode())) {
        fieldSchema = Schema.createArray(elementSchema);
    } else {
        throw new IllegalArgumentException(String.format("Unknown BigQuery Field Mode: %s", bigQueryField.getMode()));
    }
    return new Field(bigQueryField.getName(), fieldSchema, bigQueryField.getDescription(), (Object) null);
}

Example 52 with TableFieldSchema

use of com.google.api.services.bigquery.model.TableFieldSchema in project beam by apache.

the class BigQueryAvroUtils method mapTableFieldSchema.

private static Stream<TableFieldSchema> mapTableFieldSchema(TableFieldSchema fieldSchema, Schema avroSchema) {
    Field avroFieldSchema = avroSchema.getField(fieldSchema.getName());
    if (avroFieldSchema == null) {
        return Stream.empty();
    } else if (avroFieldSchema.schema().getType() != Type.RECORD) {
        return Stream.of(fieldSchema);
    }
    List<TableFieldSchema> subSchemas = fieldSchema.getFields().stream().flatMap(subSchema -> mapTableFieldSchema(subSchema, avroFieldSchema.schema())).collect(Collectors.toList());
    TableFieldSchema output = new TableFieldSchema().setCategories(fieldSchema.getCategories()).setDescription(fieldSchema.getDescription()).setFields(subSchemas).setMode(fieldSchema.getMode()).setName(fieldSchema.getName()).setType(fieldSchema.getType());
    return Stream.of(output);
}

Example 53 with TableFieldSchema

use of com.google.api.services.bigquery.model.TableFieldSchema in project beam by apache.

the class BigQueryAvroUtils method convertNullableField.

@Nullable
private static Object convertNullableField(Schema avroSchema, TableFieldSchema fieldSchema, Object v) {
    // NULLABLE fields are represented as an Avro Union of the corresponding type and "null".
    verify(avroSchema.getType() == Type.UNION, "Expected Avro schema type UNION, not %s, for BigQuery NULLABLE field %s", avroSchema.getType(), fieldSchema.getName());
    List<Schema> unionTypes = avroSchema.getTypes();
    verify(unionTypes.size() == 2, "BigQuery NULLABLE field %s should be an Avro UNION of NULL and another type, not %s", fieldSchema.getName(), unionTypes);
    if (v == null) {
        return null;
    }
    Type firstType = unionTypes.get(0).getType();
    if (!firstType.equals(Type.NULL)) {
        return convertRequiredField(firstType, unionTypes.get(0).getLogicalType(), fieldSchema, v);
    }
    return convertRequiredField(unionTypes.get(1).getType(), unionTypes.get(1).getLogicalType(), fieldSchema, v);
}

Example 54 with TableFieldSchema

use of com.google.api.services.bigquery.model.TableFieldSchema in project beam by apache.

the class BigQueryAvroUtils method convertGenericRecordToTableRow.

private static TableRow convertGenericRecordToTableRow(GenericRecord record, List<TableFieldSchema> fields) {
    TableRow row = new TableRow();
    for (TableFieldSchema subSchema : fields) {
        // Per https://cloud.google.com/bigquery/docs/reference/v2/tables#schema, the name field
        // is required, so it may not be null.
        Field field = record.getSchema().getField(subSchema.getName());
        Object convertedValue = getTypedCellValue(field.schema(), subSchema, record.get(field.name()));
        if (convertedValue != null) {
            // To match the JSON files exported by BigQuery, do not include null values in the output.
            row.set(field.name(), convertedValue);
        }
    }
    return row;
}

Example 55 with TableFieldSchema

use of com.google.api.services.bigquery.model.TableFieldSchema in project beam by apache.

the class BigQueryUtils method fromTableFieldSchemaType.

/**
 * Get the Beam {@link FieldType} from a BigQuery type name.
 *
 * <p>Supports both standard and legacy SQL types.
 *
 * @param typeName Name of the type
 * @param nestedFields Nested fields for the given type (eg. RECORD type)
 * @return Corresponding Beam {@link FieldType}
 */
@Experimental(Kind.SCHEMAS)
private static FieldType fromTableFieldSchemaType(String typeName, List<TableFieldSchema> nestedFields, SchemaConversionOptions options) {
    switch(typeName) {
        case "STRING":
            return FieldType.STRING;
        case "BYTES":
            return FieldType.BYTES;
        case "INT64":
        case "INTEGER":
            return FieldType.INT64;
        case "FLOAT64":
        case "FLOAT":
            return FieldType.DOUBLE;
        case "BOOL":
        case "BOOLEAN":
            return FieldType.BOOLEAN;
        case "NUMERIC":
            return FieldType.DECIMAL;
        case "TIMESTAMP":
            return FieldType.DATETIME;
        case "TIME":
            return FieldType.logicalType(SqlTypes.TIME);
        case "DATE":
            return FieldType.logicalType(SqlTypes.DATE);
        case "DATETIME":
            return FieldType.logicalType(SqlTypes.DATETIME);
        case "STRUCT":
        case "RECORD":
            if (options.getInferMaps() && nestedFields.size() == 2) {
                TableFieldSchema key = nestedFields.get(0);
                TableFieldSchema value = nestedFields.get(1);
                if (BIGQUERY_MAP_KEY_FIELD_NAME.equals(key.getName()) && BIGQUERY_MAP_VALUE_FIELD_NAME.equals(value.getName())) {
                    return FieldType.map(fromTableFieldSchemaType(key.getType(), key.getFields(), options), fromTableFieldSchemaType(value.getType(), value.getFields(), options));
                }
            }
            Schema rowSchema = fromTableFieldSchema(nestedFields, options);
            return FieldType.row(rowSchema);
        default:
            throw new UnsupportedOperationException("Converting BigQuery type " + typeName + " to Beam type is unsupported");
    }
}