Examples with Ddl com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl used on opensource projects

Example 16 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ChangeEventConvertorTest method cannotConvertChangeEventWithoutKeyColumnToShadowMutation.

@Test(expected = ChangeEventConvertorException.class)
public void cannotConvertChangeEventWithoutKeyColumnToShadowMutation() throws Exception {
    Ddl ddl = getTestDdl();
    JSONObject changeEvent = getTestChangeEvent("Users2");
    changeEvent.remove("last_name");
    JsonNode ce = parseChangeEvent(changeEvent.toString());
    Mutation mutation = ChangeEventConvertor.changeEventToShadowTableMutationBuilder(ddl, ce, "shadow_").build();
// Expect an Exception to be thrown as a primary key is missing.
}

Example 17 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ChangeEventConvertorTest method canConvertChangeEventToShadowMutation.

@Test
public void canConvertChangeEventToShadowMutation() throws Exception {
    Ddl ddl = getTestDdl();
    JSONObject changeEvent = getTestChangeEvent("Users2");
    JsonNode ce = parseChangeEvent(changeEvent.toString());
    Mutation mutation = ChangeEventConvertor.changeEventToShadowTableMutationBuilder(ddl, ce, "shadow_").build();
    Map<String, Value> actual = mutation.asMap();
    Map<String, Value> expected = getExpectedMapForTestChangeEvent();
    assertThat(actual, is(expected));
    assertEquals(mutation.getTable(), "shadow_Users2");
    assertEquals(mutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}

Example 18 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class OracleChangeEventContextTest method canGenerateShadowTableMutation.

@Test
public void canGenerateShadowTableMutation() throws Exception {
    // Test Ddl
    Ddl ddl = ChangeEventConvertorTest.getTestDdl();
    // Test Change Event
    JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
    changeEvent.put(DatastreamConstants.ORACLE_TIMESTAMP_KEY, eventTimestamp);
    changeEvent.put(DatastreamConstants.ORACLE_SCN_KEY, "1");
    changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.ORACLE_SOURCE_TYPE);
    ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.ORACLE_SOURCE_TYPE);
    Mutation shadowMutation = changeEventContext.getShadowTableMutation();
    Map<String, Value> actual = shadowMutation.asMap();
    // Expected result
    Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
    expected.put(DatastreamConstants.ORACLE_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
    expected.put(DatastreamConstants.ORACLE_SCN_SHADOW_INFO.getLeft(), Value.int64(1));
    // Verify if OracleChangeEventContext was actually created.
    assertThat(changeEventContext, instanceOf(OracleChangeEventContext.class));
    // Verify shadow mutation
    assertThat(actual, is(expected));
    assertEquals(shadowMutation.getTable(), "shadow_Users2");
    assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}

Example 19 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ProcessInformationSchemaIntegrationTest method canCreateShadowTablesForAllDataTables.

@Test
public void canCreateShadowTablesForAllDataTables() throws Exception {
    SpannerConfig sourceConfig = spannerServer.getSpannerConfig(testDb);
    Ddl testDdl = getTestDdlBuilder().build();
    createDb(testDdl);
    testPipeline.apply("Process Information Schema", new ProcessInformationSchema(sourceConfig, /*shouldCreateShadowTables=*/
    true, "shadow", "oracle"));
    PipelineResult testResult = testPipeline.run();
    testResult.waitUntilFinish();
    Ddl finalDdl = readDdl(testDb);
    Table shadowTable = finalDdl.table("shadow_Table");
    Table shadowTableInterleaved = finalDdl.table("shadow_Table_interleaved");
    assertNotNull(shadowTable);
    assertNotNull(shadowTableInterleaved);
    assertEquals(4, finalDdl.allTables().size());
    assertThat(shadowTable.primaryKeys(), is(testDdl.table("Table").primaryKeys()));
    assertEquals(shadowTable.columns().size(), testDdl.table("Table").primaryKeys().size() + 2);
    assertThat(shadowTableInterleaved.primaryKeys(), is(testDdl.table("Table_interleaved").primaryKeys()));
    assertEquals(shadowTableInterleaved.columns().size(), testDdl.table("Table_interleaved").primaryKeys().size() + 2);
}

Example 20 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ProcessInformationSchemaIntegrationTest method canCreateMissingShadowTables.

@Test
public void canCreateMissingShadowTables() throws Exception {
    SpannerConfig sourceConfig = spannerServer.getSpannerConfig(testDb);
    Ddl testDdl = getTestDdlBuilder().createTable("shadow_Table").column("ID").int64().endColumn().column("version").int64().endColumn().primaryKey().asc("ID").end().endTable().build();
    createDb(testDdl);
    testPipeline.apply("Process Information Schema", new ProcessInformationSchema(sourceConfig, /*shouldCreateShadowTables=*/
    true, "shadow", "oracle"));
    PipelineResult testResult = testPipeline.run();
    testResult.waitUntilFinish();
    Ddl finalDdl = readDdl(testDb);
    assertEquals(4, finalDdl.allTables().size());
    Table shadowTable = finalDdl.table("shadow_Table");
    Table shadowTableInterleaved = finalDdl.table("shadow_Table_interleaved");
    assertNotNull(shadowTable);
    assertNotNull(shadowTableInterleaved);
    assertThat(shadowTableInterleaved.primaryKeys(), is(testDdl.table("Table_interleaved").primaryKeys()));
    assertEquals(shadowTableInterleaved.columns().size(), testDdl.table("Table_interleaved").primaryKeys().size() + 2);
}