Search in sources :

Example 31 with TableSchema

use of com.google.api.services.bigquery.model.TableSchema in project beam by apache.

the class BigQueryAvroUtilsTest method testConvertBigQuerySchemaToAvroSchema.

@Test
public void testConvertBigQuerySchemaToAvroSchema() {
    TableSchema tableSchema = new TableSchema();
    tableSchema.setFields(fields);
    Schema avroSchema = BigQueryAvroUtils.toGenericAvroSchema("testSchema", tableSchema.getFields());
    assertThat(avroSchema.getField("number").schema(), equalTo(Schema.create(Type.LONG)));
    assertThat(avroSchema.getField("species").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.STRING))));
    assertThat(avroSchema.getField("quality").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.DOUBLE))));
    assertThat(avroSchema.getField("quantity").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.LONG))));
    assertThat(avroSchema.getField("birthday").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.LONG))));
    assertThat(avroSchema.getField("flighted").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.BOOLEAN))));
    assertThat(avroSchema.getField("sound").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.BYTES))));
    assertThat(avroSchema.getField("anniversaryDate").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.STRING))));
    assertThat(avroSchema.getField("anniversaryDatetime").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.STRING))));
    assertThat(avroSchema.getField("anniversaryTime").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.STRING))));
    assertThat(avroSchema.getField("scion").schema(), equalTo(Schema.createUnion(Schema.create(Type.NULL), Schema.createRecord("scion", "org.apache.beam.sdk.io.gcp.bigquery", "Translated Avro Schema for scion", false, ImmutableList.of(new Field("species", Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.STRING)), null, (Object) null))))));
    assertThat(avroSchema.getField("associates").schema(), equalTo(Schema.createArray(Schema.createRecord("associates", "org.apache.beam.sdk.io.gcp.bigquery", "Translated Avro Schema for associates", false, ImmutableList.of(new Field("species", Schema.createUnion(Schema.create(Type.NULL), Schema.create(Type.STRING)), null, (Object) null))))));
}
Also used : Field(org.apache.avro.Schema.Field) TableSchema(com.google.api.services.bigquery.model.TableSchema) TableFieldSchema(com.google.api.services.bigquery.model.TableFieldSchema) Schema(org.apache.avro.Schema) TableSchema(com.google.api.services.bigquery.model.TableSchema) Test(org.junit.Test)

Aggregations

TableSchema (com.google.api.services.bigquery.model.TableSchema)31 TableFieldSchema (com.google.api.services.bigquery.model.TableFieldSchema)20 TableRow (com.google.api.services.bigquery.model.TableRow)18 JsonSchemaToTableSchema (org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.JsonSchemaToTableSchema)13 Test (org.junit.Test)13 TableReference (com.google.api.services.bigquery.model.TableReference)12 Pipeline (org.apache.beam.sdk.Pipeline)12 ArrayList (java.util.ArrayList)10 BigQueryHelpers.toJsonString (org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.toJsonString)9 TestPipeline (org.apache.beam.sdk.testing.TestPipeline)8 Table (com.google.api.services.bigquery.model.Table)7 BigQueryHelpers.createTempTableReference (org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.createTempTableReference)7 PipelineOptions (org.apache.beam.sdk.options.PipelineOptions)7 HashBasedTable (com.google.common.collect.HashBasedTable)6 JobStatus (com.google.api.services.bigquery.model.JobStatus)5 JobStatistics (com.google.api.services.bigquery.model.JobStatistics)4 JobStatistics4 (com.google.api.services.bigquery.model.JobStatistics4)4 Path (java.nio.file.Path)4 Map (java.util.Map)4 Job (com.google.api.services.bigquery.model.Job)3