Search in sources :

Example 91 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class CopyDbTest method changeStreams.

@Test
public void changeStreams() throws Exception {
    Ddl ddl = Ddl.builder().createTable("T1").endTable().createTable("T2").column("key").int64().endColumn().column("c1").int64().endColumn().column("c2").string().max().endColumn().primaryKey().asc("key").end().endTable().createTable("T3").endTable().createChangeStream("ChangeStreamAll").forClause("FOR ALL").options(ImmutableList.of("retention_period=\"7d\"", "value_capture_type=\"OLD_AND_NEW_VALUES\"")).endChangeStream().createChangeStream("ChangeStreamEmpty").endChangeStream().createChangeStream("ChangeStreamTableColumns").forClause("FOR `T1`, `T2`(`c1`, `c2`), `T3`()").endChangeStream().build();
    createAndPopulate(ddl, 0);
    runTest();
}
Also used : Ddl(com.google.cloud.teleport.spanner.ddl.Ddl) Test(org.junit.Test)

Example 92 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class CopyDbTest method pgForeignKeys.

@Test
public void pgForeignKeys() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder(Dialect.POSTGRESQL).createTable("Ref").column("id1").pgInt8().endColumn().column("id2").pgInt8().endColumn().primaryKey().asc("id1").asc("id2").end().endTable().createTable("Child").column("id1").pgInt8().endColumn().column("id2").pgInt8().endColumn().column("id3").pgInt8().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("Ref").foreignKeys(ImmutableList.of("ALTER TABLE \"Child\" ADD CONSTRAINT \"fk1\" FOREIGN KEY (\"id1\") REFERENCES" + " \"Ref\" (\"id1\")", "ALTER TABLE \"Child\" ADD CONSTRAINT \"fk2\" FOREIGN KEY (\"id2\") REFERENCES" + " \"Ref\" (\"id2\")", "ALTER TABLE \"Child\" ADD CONSTRAINT \"fk3\" FOREIGN KEY (\"id2\") REFERENCES" + " \"Ref\" (\"id2\")", "ALTER TABLE \"Child\" ADD CONSTRAINT \"fk4\" FOREIGN KEY (\"id2\", \"id1\") " + "REFERENCES \"Ref\" (\"id2\", \"id1\")")).endTable().build();
    // spotless:on
    createAndPopulate(ddl, 100);
    runTest(Dialect.POSTGRESQL);
}
Also used : Ddl(com.google.cloud.teleport.spanner.ddl.Ddl) Test(org.junit.Test)

Example 93 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class CopyDbTest method pgDatabaseOptions.

@Test
public void pgDatabaseOptions() throws Exception {
    Ddl.Builder ddlBuilder = Ddl.builder(Dialect.POSTGRESQL);
    // Table Content
    // spotless:off
    ddlBuilder.createTable("Users").column("first_name").pgVarchar().max().endColumn().column("last_name").pgVarchar().size(5).endColumn().column("age").pgInt8().endColumn().primaryKey().asc("first_name").asc("last_name").end().endTable().createTable("EmploymentData").column("first_name").pgVarchar().max().endColumn().column("last_name").pgVarchar().size(5).endColumn().column("id").pgInt8().notNull().endColumn().column("age").pgInt8().endColumn().column("address").pgVarchar().max().endColumn().primaryKey().asc("first_name").asc("last_name").asc("id").end().interleaveInParent("Users").onDeleteCascade().endTable();
    // spotless:on
    // Allowed and well-formed database option
    List<Export.DatabaseOption> dbOptionList = new ArrayList<>();
    dbOptionList.add(Export.DatabaseOption.newBuilder().setOptionName("version_retention_period").setOptionValue("'6d'").build());
    // Disallowed database option
    dbOptionList.add(Export.DatabaseOption.newBuilder().setOptionName("optimizer_version").setOptionValue("1").build());
    // Misformed database option
    dbOptionList.add(Export.DatabaseOption.newBuilder().setOptionName("123version").setOptionValue("xyz").build());
    ddlBuilder.mergeDatabaseOptions(dbOptionList);
    Ddl ddl = ddlBuilder.build();
    createAndPopulate(ddl, 100);
    runTest(Dialect.POSTGRESQL);
    Ddl destinationDdl = readDdl(destinationDb, Dialect.POSTGRESQL);
    List<String> destDbOptions = destinationDdl.setOptionsStatements(destinationDb);
    assertThat(destDbOptions.size(), is(1));
    assertThat(destDbOptions.get(0), is("ALTER DATABASE \"" + destinationDb + "\" SET spanner.version_retention_period = '6d'"));
}
Also used : ArrayList(java.util.ArrayList) Ddl(com.google.cloud.teleport.spanner.ddl.Ddl) Test(org.junit.Test)

Example 94 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class CopyDbTest method allPgTypesSchema.

@Test
public void allPgTypesSchema() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder(Dialect.POSTGRESQL).createTable("Users").column("first_name").pgVarchar().max().endColumn().column("last_name").pgVarchar().size(5).endColumn().column("age").pgInt8().endColumn().primaryKey().asc("first_name").asc("last_name").end().endTable().createTable("AllTYPES").column("id").pgInt8().notNull().endColumn().column("first_name").pgVarchar().max().endColumn().column("last_name").pgVarchar().size(5).endColumn().column("bool_field").pgBool().endColumn().column("int_field").pgInt8().endColumn().column("float_field").pgFloat8().endColumn().column("string_field").pgText().endColumn().column("bytes_field").pgBytea().endColumn().column("timestamp_field").pgTimestamptz().endColumn().column("numeric_field").pgNumeric().endColumn().column("date_field").pgDate().endColumn().column("arr_bool_field").type(Type.pgArray(Type.pgBool())).endColumn().column("arr_int_field").type(Type.pgArray(Type.pgInt8())).endColumn().column("arr_float_field").type(Type.pgArray(Type.pgFloat8())).endColumn().column("arr_string_field").type(Type.pgArray(Type.pgVarchar())).max().endColumn().column("arr_bytes_field").type(Type.pgArray(Type.pgBytea())).max().endColumn().column("arr_timestamp_field").type(Type.pgArray(Type.pgTimestamptz())).endColumn().column("arr_date_field").type(Type.pgArray(Type.pgDate())).endColumn().column("arr_numeric_field").type(Type.pgArray(Type.pgNumeric())).endColumn().primaryKey().asc("first_name").asc("last_name").asc("id").asc("float_field").end().interleaveInParent("Users").onDeleteCascade().endTable().build();
    // spotless:on
    createAndPopulate(ddl, 100);
    runTest(Dialect.POSTGRESQL);
}
Also used : Ddl(com.google.cloud.teleport.spanner.ddl.Ddl) Test(org.junit.Test)

Example 95 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class CopyDbTest method emptyPgDb.

@Test
public void emptyPgDb() throws Exception {
    Ddl ddl = Ddl.builder(Dialect.POSTGRESQL).build();
    createAndPopulate(ddl, 0);
    runTest(Dialect.POSTGRESQL);
}
Also used : Ddl(com.google.cloud.teleport.spanner.ddl.Ddl) Test(org.junit.Test)

Aggregations

Ddl (com.google.cloud.teleport.spanner.ddl.Ddl)109 Test (org.junit.Test)91 Schema (org.apache.avro.Schema)34 GenericRecord (org.apache.avro.generic.GenericRecord)19 List (java.util.List)18 Struct (com.google.cloud.spanner.Struct)14 Collectors (java.util.stream.Collectors)14 KV (org.apache.beam.sdk.values.KV)14 SpannerTableFilter.getFilteredTables (com.google.cloud.teleport.spanner.SpannerTableFilter.getFilteredTables)12 Type (com.google.cloud.teleport.spanner.common.Type)12 Path (java.nio.file.Path)12 Collections (java.util.Collections)12 ImmutableList (com.google.common.collect.ImmutableList)11 IOException (java.io.IOException)11 Assert.assertEquals (org.junit.Assert.assertEquals)11 ReadImportManifest (com.google.cloud.teleport.spanner.TextImportTransform.ReadImportManifest)10 ResolveDataFiles (com.google.cloud.teleport.spanner.TextImportTransform.ResolveDataFiles)10 BufferedWriter (java.io.BufferedWriter)10 Charset (java.nio.charset.Charset)10 RunWith (org.junit.runner.RunWith)9