Examples with Ddl com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl used on opensource projects

Example 21 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ProcessInformationSchemaIntegrationTest method canFlagProtectShadowTableCreation.

@Test
public void canFlagProtectShadowTableCreation() throws Exception {
    SpannerConfig sourceConfig = spannerServer.getSpannerConfig(testDb);
    Ddl testDdl = getTestDdlBuilder().build();
    createDb(testDdl);
    testPipeline.apply("Read Information Schema", new ProcessInformationSchema(sourceConfig, /*shouldCreateShadowTables=*/
    false, "shadow", "oracle"));
    PipelineResult testResult = testPipeline.run();
    testResult.waitUntilFinish();
    Ddl finalDdl = readDdl(testDb);
    Table shadowTable = finalDdl.table("shadow_Table");
    Table shadowTableInterleaved = finalDdl.table("shadow_Table_interleaved");
    assertNull(shadowTable);
    assertNull(shadowTableInterleaved);
    assertEquals(2, finalDdl.allTables().size());
}

Example 22 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ShadowTableCreatorTest method canConstructShadowTableForMySql.

@Test
public void canConstructShadowTableForMySql() {
    Ddl testDdl = ProcessInformationSchemaTest.getTestDdl();
    ShadowTableCreator shadowTableCreator = new ShadowTableCreator("mysql", "shadow_");
    Table shadowTable = shadowTableCreator.constructShadowTable(testDdl, "Users_interleaved");
    /* Verify
     * (1) name of shadow table
     * (2) primary keys columns are same as data tables
     * (3) Has mysql sequence information in addition to primary keys columns
     */
    assertEquals(shadowTable.name(), "shadow_Users_interleaved");
    assertThat(shadowTable.primaryKeys(), is(testDdl.table("Users_interleaved").primaryKeys()));
    Set<String> columns = shadowTable.columns().stream().map(c -> c.name()).collect(Collectors.toSet());
    Set<String> expectedColumns = testDdl.table("Users_interleaved").primaryKeys().stream().map(c -> c.name()).collect(Collectors.toSet());
    expectedColumns.add("timestamp");
    expectedColumns.add("log_file");
    expectedColumns.add("log_position");
    assertThat(columns, is(expectedColumns));
}

Example 23 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ShadowTableCreator method constructShadowTable.

/*
   * Constructs a shadow table for a data table in the information schema.
   * Note: Shadow tables for interleaved tables are not interleaved to
   * their shadow parent table.
   */
Table constructShadowTable(Ddl informationSchema, String dataTableName) {
    // Create a new shadow table with the given prefix.
    Table.Builder shadowTableBuilder = Table.builder();
    String shadowTableName = shadowTablePrefix + dataTableName;
    shadowTableBuilder.name(shadowTableName);
    // Add key columns from the data table to the shadow table builder.
    Table dataTable = informationSchema.table(dataTableName);
    Set<String> primaryKeyColNames = dataTable.primaryKeys().stream().map(k -> k.name()).collect(Collectors.toSet());
    List<Column> primaryKeyCols = dataTable.columns().stream().filter(col -> primaryKeyColNames.contains(col.name())).collect(Collectors.toList());
    for (Column col : primaryKeyCols) {
        shadowTableBuilder.addColumn(col);
    }
    // Add primary key constraints.
    for (IndexColumn keyColumn : dataTable.primaryKeys()) {
        if (keyColumn.order() == IndexColumn.Order.ASC) {
            shadowTableBuilder.primaryKey().asc(keyColumn.name()).end();
        } else if (keyColumn.order() == IndexColumn.Order.DESC) {
            shadowTableBuilder.primaryKey().desc(keyColumn.name()).end();
        }
    }
    // Add extra column to track ChangeEventSequence information
    addChangeEventSequenceColumns(shadowTableBuilder);
    return shadowTableBuilder.build();
}

Example 24 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class MySqlChangeEventContextTest method canGenerateShadowTableMutation.

@Test
public void canGenerateShadowTableMutation() throws Exception {
    long eventTimestamp = 1615159728L;
    // Test Ddl
    Ddl ddl = ChangeEventConvertorTest.getTestDdl();
    // Test Change Event
    JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
    changeEvent.put(DatastreamConstants.MYSQL_TIMESTAMP_KEY, eventTimestamp);
    changeEvent.put(DatastreamConstants.MYSQL_LOGFILE_KEY, "file1.log");
    changeEvent.put(DatastreamConstants.MYSQL_LOGPOSITION_KEY, 1L);
    changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.MYSQL_SOURCE_TYPE);
    ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.MYSQL_SOURCE_TYPE);
    Mutation shadowMutation = changeEventContext.getShadowTableMutation();
    Map<String, Value> actual = shadowMutation.asMap();
    // Expected result
    Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
    expected.put(DatastreamConstants.MYSQL_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
    expected.put(DatastreamConstants.MYSQL_LOGFILE_SHADOW_INFO.getLeft(), Value.string("file1.log"));
    expected.put(DatastreamConstants.MYSQL_LOGPOSITION_SHADOW_INFO.getLeft(), Value.int64(1));
    // Verify if MySqlChangeEventContext was actually created.
    assertThat(changeEventContext, instanceOf(MySqlChangeEventContext.class));
    // Verify shadow mutation
    assertThat(actual, is(expected));
    assertEquals(shadowMutation.getTable(), "shadow_Users2");
    assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}

Example 25 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class MySqlChangeEventContextTest method canGenerateShadowTableMutationForBackfillEventsWithMissingSortOrderKeys.

@Test
public void canGenerateShadowTableMutationForBackfillEventsWithMissingSortOrderKeys() throws Exception {
    long eventTimestamp = 1615159728L;
    // Test Ddl
    Ddl ddl = ChangeEventConvertorTest.getTestDdl();
    // Test Change Event which does not contain sort order fields like log file and log position.
    JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
    changeEvent.put(DatastreamConstants.MYSQL_TIMESTAMP_KEY, eventTimestamp);
    changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.MYSQL_SOURCE_TYPE);
    ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.MYSQL_SOURCE_TYPE);
    Mutation shadowMutation = changeEventContext.getShadowTableMutation();
    Map<String, Value> actual = shadowMutation.asMap();
    // Expected result
    Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
    expected.put(DatastreamConstants.MYSQL_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
    expected.put(DatastreamConstants.MYSQL_LOGFILE_SHADOW_INFO.getLeft(), Value.string(""));
    expected.put(DatastreamConstants.MYSQL_LOGPOSITION_SHADOW_INFO.getLeft(), Value.int64(-1));
    // Verify if MySqlChangeEventContext was actually created.
    assertThat(changeEventContext, instanceOf(MySqlChangeEventContext.class));
    // Verify shadow mutation
    assertThat(actual, is(expected));
    assertEquals(shadowMutation.getTable(), "shadow_Users2");
    assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}