Examples with Ddl com.google.cloud.teleport.spanner.ddl.Ddl used on opensource projects

Example 11 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportFullDbWithFlagFalse.

/* Validates that pipeline executes full-db export when --tableNames is not filled and
   * --shouldExportRelatedTables is set to false (either intentionally or by default) */
@Test
public void exportFullDbWithFlagFalse() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("table_a").column("id1").int64().endColumn().column("id2").int64().endColumn().primaryKey().asc("id1").asc("id2").end().endTable().createTable("table_b").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_c").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_b").endTable().build();
    // spotless:on
    // Add to referencedTable field (i.e. `table_c` would have a foreign key constraint
    // referencing `table_a` )
    ddl.addNewReferencedTable("table_c", "table_a");
    createAndPopulate(ddl, /* numBatches = */
    100);
    // Export the entire database (without setting --tablesName)
    spannerServer.createDatabase(destDbPrefix + chkptFive, Collections.emptyList());
    exportAndImportDb(sourceDb, destDbPrefix + chkptFive, chkptFive, // --tableNames would not be set, defaults to an empty string
    "", /* relatedTables =*/
    false, exportPipeline, importPipeline);
    // Compare the tables in the ddl to ensure all original tables were re-created during the import
    compareExpectedTables(destDbPrefix + chkptFive, ImmutableList.of(tableA, tableB, tableC));
    // Check to see selected tables exported with data and and unselected tables did not
    List<String> exportTables = ImmutableList.of(tableA, tableB, tableC);
    List<String> unselectedTables = Collections.emptyList();
    compareExpectedTableRows(destDbPrefix + chkptFive, exportTables, unselectedTables);
}

Example 12 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportDbWithoutTableNamesAndFlag_exportsFullDb.

/* Validates that pipeline executes full-db export when --tableNames and
   * --shouldExportRelatedTables paramters are not filled */
@Test
public void exportDbWithoutTableNamesAndFlag_exportsFullDb() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("table_a").column("id1").int64().endColumn().column("id2").int64().endColumn().primaryKey().asc("id1").asc("id2").end().endTable().createTable("table_b").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_c").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_b").endTable().build();
    // spotless:on
    // Add to referencedTable field (i.e. `table_c` would have a foreign key constraint
    // referencing `table_a` )
    ddl.addNewReferencedTable("table_c", "table_a");
    createAndPopulate(ddl, /* numBatches = */
    100);
    // Export the entire database (without setting --tablesName or --shouldExportRelatedTables)
    spannerServer.createDatabase(destDbPrefix + chkptOne, Collections.emptyList());
    exportAndImportDb(sourceDb, destDbPrefix + chkptOne, chkptOne, // --tableNames would not be set, defaults to an empty string
    "", /* relatedTables =*/
    false, exportPipeline, importPipeline);
    // Compare the tables in the ddl to ensure all original tables were re-created during the import
    compareExpectedTables(destDbPrefix + chkptOne, ImmutableList.of(tableA, tableB, tableC));
    // Check to see selected tables exported with data and and unselected tables did not
    List<String> exportTables = ImmutableList.of(tableA, tableB, tableC);
    List<String> unselectedTables = Collections.emptyList();
    compareExpectedTableRows(destDbPrefix + chkptOne, exportTables, unselectedTables);
}

Example 13 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportSelectedAndNecessaryTables.

/* Validates that pipeline executes table level export when --tableNames is provided,
   * --shouldExportRelatedTables is set to true, and additional tables need to be exported */
@Test
public void exportSelectedAndNecessaryTables() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("table_a").column("id1").int64().endColumn().column("id2").int64().endColumn().primaryKey().asc("id1").asc("id2").end().endTable().createTable("table_b").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_c").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_b").foreignKeys(ImmutableList.of("ALTER TABLE `table_c` ADD CONSTRAINT `fk1` FOREIGN KEY (`id1`) REFERENCES" + " `table_b` (`id1`)")).endTable().createTable("table_d").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().build();
    // spotless:on
    createAndPopulate(ddl, /* numBatches = */
    100);
    // Export the single table along with it's required tables
    spannerServer.createDatabase(destDbPrefix + chkptSix, Collections.emptyList());
    exportAndImportDb(sourceDb, destDbPrefix + chkptSix, chkptSix, // --tableNames would be given tableC
    tableC, /* relatedTables =*/
    true, exportPipeline, importPipeline);
    // Compare the tables in the ddl to ensure all original tables were re-created during the import
    compareExpectedTables(destDbPrefix + chkptSix, ImmutableList.of(tableA, tableB, tableC, tableD));
    // Check to see selected tables exported with data and and unselected tables did not
    List<String> exportTables = ImmutableList.of(tableB, tableC);
    List<String> unselectedTables = ImmutableList.of(tableA, tableD);
    compareExpectedTableRows(destDbPrefix + chkptSix, exportTables, unselectedTables);
}

Example 14 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportTableWithoutRelatedTablesAndWithoutFlag_exportsSelectedTable.

/* Validates that pipeline exports single table that has no related tables when
   * --shouldExportRelatedTables parameter is not filled */
@Test
public void exportTableWithoutRelatedTablesAndWithoutFlag_exportsSelectedTable() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("table_a").column("id1").int64().endColumn().column("id2").int64().endColumn().primaryKey().asc("id1").asc("id2").end().endTable().createTable("table_b").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_a").endTable().createTable("table_c").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().build();
    // spotless:on
    createAndPopulate(ddl, /* numBatches = */
    100);
    // Export the single disconnected table database (without --shouldExportRelatedTables)
    spannerServer.createDatabase(destDbPrefix + chkptTwo, Collections.emptyList());
    exportAndImportDb(sourceDb, destDbPrefix + chkptTwo, chkptTwo, tableC, /* relatedTables =*/
    false, exportPipeline, importPipeline);
    // Compare the tables in the ddl to ensure all original tables were re-created during the import
    compareExpectedTables(destDbPrefix + chkptTwo, ImmutableList.of(tableA, tableB, tableC));
    // Check to see selected tables exported with data and and unselected tables did not
    List<String> exportTables = ImmutableList.of(tableC);
    List<String> unselectedTables = ImmutableList.of(tableA, tableB);
    compareExpectedTableRows(destDbPrefix + chkptTwo, exportTables, unselectedTables);
}

Example 15 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method exportNonExistentTable_stopsPipelineExecution.

/* Validates that pipeline execution fails when --tableNames is given a table that doesn't
   * exist in the database */
@Test
public void exportNonExistentTable_stopsPipelineExecution() throws Exception {
    // spotless:off
    Ddl ddl = Ddl.builder().createTable("table_a").column("id1").int64().endColumn().column("id2").int64().endColumn().primaryKey().asc("id1").asc("id2").end().endTable().createTable("table_b").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().endTable().createTable("table_c").column("id1").int64().endColumn().column("id2").int64().endColumn().column("id3").int64().endColumn().primaryKey().asc("id1").asc("id2").asc("id3").end().interleaveInParent("table_b").endTable().build();
    // spotless:on
    // Add to referencedTable field (i.e. `table_c` would have a foreign key constraint
    // referencing `table_a`)
    ddl.addNewReferencedTable("table_c", "table_a");
    createAndPopulate(ddl, /* numBatches = */
    100);
    // Expected PipelineExecutionException caused by Exception:
    // Attempt to export a non existent table 'table_d'.
    spannerServer.createDatabase(destDbPrefix + chkptEight, Collections.emptyList());
    assertThrows(PipelineExecutionException.class, () -> exportAndImportDb(sourceDb, destDbPrefix + chkptEight, chkptEight, tableD, /* relatedTables =*/
    false, exportPipeline, importPipeline));
}