Examples with Ddl com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl used on opensource projects

Example 26 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class MySqlChangeEventContextTest method canGenerateShadowTableMutationForBackfillEvents.

@Test
public void canGenerateShadowTableMutationForBackfillEvents() throws Exception {
    long eventTimestamp = 1615159728L;
    // Test Ddl
    Ddl ddl = ChangeEventConvertorTest.getTestDdl();
    // Test Change Event
    JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
    changeEvent.put(DatastreamConstants.MYSQL_TIMESTAMP_KEY, eventTimestamp);
    changeEvent.put(DatastreamConstants.MYSQL_LOGFILE_KEY, JSONObject.NULL);
    changeEvent.put(DatastreamConstants.MYSQL_LOGPOSITION_KEY, JSONObject.NULL);
    changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.MYSQL_SOURCE_TYPE);
    ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.MYSQL_SOURCE_TYPE);
    Mutation shadowMutation = changeEventContext.getShadowTableMutation();
    Map<String, Value> actual = shadowMutation.asMap();
    // Expected result
    Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
    expected.put(DatastreamConstants.MYSQL_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
    expected.put(DatastreamConstants.MYSQL_LOGFILE_SHADOW_INFO.getLeft(), Value.string(""));
    expected.put(DatastreamConstants.MYSQL_LOGPOSITION_SHADOW_INFO.getLeft(), Value.int64(-1));
    // Verify if MySqlChangeEventContext was actually created.
    assertThat(changeEventContext, instanceOf(MySqlChangeEventContext.class));
    // Verify shadow mutation
    assertThat(actual, is(expected));
    assertEquals(shadowMutation.getTable(), "shadow_Users2");
    assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}

Example 27 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class OracleChangeEventContextTest method canGenerateShadowTableMutationForBackfillEvent.

@Test
public void canGenerateShadowTableMutationForBackfillEvent() throws Exception {
    // Test Ddl
    Ddl ddl = ChangeEventConvertorTest.getTestDdl();
    // Test Change Event
    JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
    changeEvent.put(DatastreamConstants.ORACLE_TIMESTAMP_KEY, eventTimestamp);
    changeEvent.put(DatastreamConstants.ORACLE_SCN_KEY, JSONObject.NULL);
    changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.ORACLE_SOURCE_TYPE);
    ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.ORACLE_SOURCE_TYPE);
    Mutation shadowMutation = changeEventContext.getShadowTableMutation();
    Map<String, Value> actual = shadowMutation.asMap();
    // Expected result
    Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
    expected.put(DatastreamConstants.ORACLE_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
    expected.put(DatastreamConstants.ORACLE_SCN_SHADOW_INFO.getLeft(), Value.int64(-1));
    // Verify if OracleChangeEventContext was actually created.
    assertThat(changeEventContext, instanceOf(OracleChangeEventContext.class));
    // Verify shadow mutation
    assertThat(actual, is(expected));
    assertEquals(shadowMutation.getTable(), "shadow_Users2");
    assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}

Example 28 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class OracleChangeEventContextTest method canGenerateShadowTableMutationForBackfillEventWithMissingKeys.

@Test
public void canGenerateShadowTableMutationForBackfillEventWithMissingKeys() throws Exception {
    // Test Ddl
    Ddl ddl = ChangeEventConvertorTest.getTestDdl();
    // Test Change Event
    JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
    changeEvent.put(DatastreamConstants.ORACLE_TIMESTAMP_KEY, eventTimestamp);
    changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.ORACLE_SOURCE_TYPE);
    ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.ORACLE_SOURCE_TYPE);
    Mutation shadowMutation = changeEventContext.getShadowTableMutation();
    Map<String, Value> actual = shadowMutation.asMap();
    // Expected result
    Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
    expected.put(DatastreamConstants.ORACLE_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
    expected.put(DatastreamConstants.ORACLE_SCN_SHADOW_INFO.getLeft(), Value.int64(-1));
    // Verify if OracleChangeEventContext was actually created.
    assertThat(changeEventContext, instanceOf(OracleChangeEventContext.class));
    // Verify shadow mutation
    assertThat(actual, is(expected));
    assertEquals(shadowMutation.getTable(), "shadow_Users2");
    assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}

Example 29 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ProcessInformationSchemaIntegrationTest method readDdl.

private Ddl readDdl(String db) {
    DatabaseClient dbClient = spannerServer.getDbClient(db);
    Ddl ddl;
    try (ReadOnlyTransaction ctx = dbClient.readOnlyTransaction()) {
        ddl = new InformationSchemaScanner(ctx).scan();
    }
    return ddl;
}

Example 30 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ShadowTableCreatorTest method canConstructShadowTableForOracle.

@Test
public void canConstructShadowTableForOracle() {
    Ddl testDdl = ProcessInformationSchemaTest.getTestDdl();
    ShadowTableCreator shadowTableCreator = new ShadowTableCreator("oracle", "shadow_");
    Table shadowTable = shadowTableCreator.constructShadowTable(testDdl, "Users_interleaved");
    /* Verify
     * (1) name of shadow table
     * (2) primary keys columns are same as data tables
     * (3) Has oracle sequence information column in addition to primary keys columns
     */
    assertEquals(shadowTable.name(), "shadow_Users_interleaved");
    assertThat(shadowTable.primaryKeys(), is(testDdl.table("Users_interleaved").primaryKeys()));
    Set<String> columns = shadowTable.columns().stream().map(c -> c.name()).collect(Collectors.toSet());
    Set<String> expectedColumns = testDdl.table("Users_interleaved").primaryKeys().stream().map(c -> c.name()).collect(Collectors.toSet());
    expectedColumns.add("timestamp");
    expectedColumns.add("scn");
    assertThat(columns, is(expectedColumns));
}