Examples with Ddl com.google.cloud.teleport.spanner.ddl.Ddl used on opensource projects

Example 61 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method compareExpectedTables.

/* Compares a given List of strings for the expected tables in a database (in alphabetical order)
   * and compares that list to the list of actual tables in that database */
private void compareExpectedTables(String sourceDb, List<String> expectedTables) {
    Ddl sourceDdl = readDdl(sourceDb);
    List<String> tables = getTableNamesFromDdl(sourceDdl);
    Collections.sort(tables);
    assertEquals(expectedTables, tables);
}

Example 62 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportTableWithRelatedTablesAndWithoutFlag_stopsPipelineExecution.

/* Validates that pipeline execution fails when --tableNames is provided and
   * --shouldExportRelatedTables is not filled/set to false, but additional tables
   * need to be exported */
@Test
public void exportTableWithRelatedTablesAndWithoutFlag_stopsPipelineExecution() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("table_a").column("id1").int64().endColumn().column("id2").int64().endColumn().primaryKey().asc("id1").asc("id2").end().endTable().createTable("table_b").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_c").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_b").foreignKeys(ImmutableList.of("ALTER TABLE `table_c` ADD CONSTRAINT `fk_table_b` FOREIGN KEY (`id1`)" + " REFERENCES `table_b` (`id1`)")).endTable().createTable("table_d").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_e").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_f").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_e").foreignKeys(ImmutableList.of("ALTER TABLE `table_f` ADD CONSTRAINT `fk_table_f` FOREIGN KEY (`id2`)" + " REFERENCES `table_e` (`id2`)")).endTable().createTable("table_g").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_h").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_i").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_h").endTable().createTable("table_j").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_i").endTable().createTable("table_k").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_l").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().build();
    // spotless:apply
    createAndPopulate(ddl, /* numBatches = */
    100);
    // Expected PipelineExecutionException caused by Exception:
    // Attempt to export a single table that requires additional related tables
    // (without --shouldExportRelatedTables set/setting --shouldExportRelatedTables true)
    spannerServer.createDatabase(destDbPrefix + chkptThree, Collections.emptyList());
    Exception exception = assertThrows(PipelineExecutionException.class, () -> exportAndImportDb(sourceDb, destDbPrefix + chkptThree, chkptThree, String.join(",", ImmutableList.of(tableA, tableC, tableF, tableJ)), /* relatedTables =*/
    false, exportPipeline, importPipeline));
    List<String> missingTables = ImmutableList.of(tableB, tableE, tableH, tableI);
    assertEquals("java.lang.Exception: Attempted to export table(s) requiring parent and/or foreign keys" + " tables without setting the shouldExportRelatedTables parameter. Set" + " --shouldExportRelatedTables=true to export all necessary tables, or add " + String.join(", ", missingTables) + " to --tableNames.", exception.getMessage());
}

Example 63 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportSingleTable.

/* Validates behavior of single table database exporting */
@Test
public void exportSingleTable() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("Users").column("first_name").string().max().endColumn().column("last_name").string().size(5).endColumn().column("age").int64().endColumn().primaryKey().asc("first_name").desc("last_name").end().endTable().createTable("AllTYPES").column("first_name").string().max().endColumn().column("last_name").string().size(5).endColumn().column("id").int64().notNull().endColumn().column("bool_field").bool().endColumn().column("int64_field").int64().endColumn().column("float64_field").float64().endColumn().column("string_field").string().max().endColumn().column("bytes_field").bytes().max().endColumn().column("timestamp_field").timestamp().endColumn().column("date_field").date().endColumn().column("arr_bool_field").type(Type.array(Type.bool())).endColumn().column("arr_int64_field").type(Type.array(Type.int64())).endColumn().column("arr_float64_field").type(Type.array(Type.float64())).endColumn().column("arr_string_field").type(Type.array(Type.string())).max().endColumn().column("arr_bytes_field").type(Type.array(Type.bytes())).max().endColumn().column("arr_timestamp_field").type(Type.array(Type.timestamp())).endColumn().column("arr_date_field").type(Type.array(Type.date())).endColumn().primaryKey().asc("first_name").desc("last_name").asc("id").end().endTable().build();
    // spotless:on
    createAndPopulate(ddl, 100);
    // Export and import the table 'Users' from the database only
    spannerServer.createDatabase(destDbPrefix + usersChkpt, Collections.emptyList());
    exportAndImportDb(sourceDb, destDbPrefix + usersChkpt, usersChkpt, usersTable, /* relatedTables =*/
    false, exportPipeline, importPipeline);
    // Compare the tables in the ddl to ensure all original tables were re-created during the import
    compareExpectedTables(destDbPrefix + usersChkpt, ImmutableList.of(allTypesTable, usersTable));
    // Check to see selected tables exported with data and and unselected tables did not
    List<String> exportTables = ImmutableList.of(usersTable);
    List<String> unselectedTables = ImmutableList.of(allTypesTable);
    compareExpectedTableRows(destDbPrefix + usersChkpt, exportTables, unselectedTables);
}

Example 64 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportSingleEmptyTable.

/* Validates behavior of exporting a single, empty table from a database */
@Test
public void exportSingleEmptyTable() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("Users").column("first_name").string().max().endColumn().column("last_name").string().size(5).endColumn().column("age").int64().endColumn().primaryKey().asc("first_name").desc("last_name").end().endTable().createTable("AllTYPES").column("first_name").string().max().endColumn().column("last_name").string().size(5).endColumn().column("id").int64().notNull().endColumn().column("bool_field").bool().endColumn().column("int64_field").int64().endColumn().column("float64_field").float64().endColumn().column("string_field").string().max().endColumn().column("bytes_field").bytes().max().endColumn().column("timestamp_field").timestamp().endColumn().column("date_field").date().endColumn().column("arr_bool_field").type(Type.array(Type.bool())).endColumn().column("arr_int64_field").type(Type.array(Type.int64())).endColumn().column("arr_float64_field").type(Type.array(Type.float64())).endColumn().column("arr_string_field").type(Type.array(Type.string())).max().endColumn().column("arr_bytes_field").type(Type.array(Type.bytes())).max().endColumn().column("arr_timestamp_field").type(Type.array(Type.timestamp())).endColumn().column("arr_date_field").type(Type.array(Type.date())).endColumn().primaryKey().asc("first_name").desc("last_name").asc("id").end().endTable().build();
    // spotless:on
    createAndPopulate(ddl, 100);
    // Add empty table.
    // spotless:off
    Ddl ddlEmptyTable = Ddl.builder().createTable("empty_table").column("first").string().max().endColumn().column("second").string().size(5).endColumn().column("value").int64().endColumn().primaryKey().asc("first").desc("second").end().endTable().build();
    // spotless:on
    spannerServer.updateDatabase(sourceDb, ddlEmptyTable.createTableStatements());
    // Export an empty table from a database
    spannerServer.createDatabase(destDbPrefix + emptyChkpt, Collections.emptyList());
    exportAndImportDb(sourceDb, destDbPrefix + emptyChkpt, emptyChkpt, emptyTable, /* relatedTables =*/
    false, exportPipeline, importPipeline);
    // Compare the tables in the ddl to ensure all original tables were re-created during the import
    compareExpectedTables(destDbPrefix + emptyChkpt, ImmutableList.of(allTypesTable, usersTable, emptyTable));
    // Check to see selected tables exported with data and and unselected tables did not
    List<String> exportTables = Collections.emptyList();
    List<String> unselectedTables = ImmutableList.of(allTypesTable, usersTable, emptyTable);
    compareExpectedTableRows(destDbPrefix + emptyChkpt, exportTables, unselectedTables);
}

Example 65 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportTimestampTest method compareDbs.

private void compareDbs(String sourceDb, String destDb, TestPipeline comparePipeline, Dialect dialect) {
    SpannerConfig sourceConfig = spannerServer.getSpannerConfig(sourceDb);
    SpannerConfig copyConfig = spannerServer.getSpannerConfig(destDb);
    PCollection<Long> mismatchCount = comparePipeline.apply("Compare", new CompareDatabases(sourceConfig, copyConfig));
    PAssert.that(mismatchCount).satisfies((x) -> {
        assertEquals(Lists.newArrayList(x), Lists.newArrayList(0L));
        return null;
    });
    PipelineResult compareResult = comparePipeline.run();
    compareResult.waitUntilFinish();
    Ddl sourceDdl = readDdl(sourceDb, dialect);
    Ddl destinationDdl = readDdl(destDb, dialect);
    assertThat(sourceDdl.prettyPrint(), equalToCompressingWhiteSpace(destinationDdl.prettyPrint()));
}