Examples with Ddl com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl used on opensource projects

Example 31 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class SpannerStreamingWriteIntegrationTest method constructAndRunPipeline.

private void constructAndRunPipeline(PCollection<FailsafeElement<String, String>> jsonRecords) {
    String shadowTablePrefix = "shadow";
    SpannerConfig sourceConfig = spannerServer.getSpannerConfig(testDb);
    PCollection<Ddl> ddl = testPipeline.apply("Process Information Schema", new ProcessInformationSchema(sourceConfig, true, shadowTablePrefix, "oracle"));
    PCollectionView<Ddl> ddlView = ddl.apply("Cloud Spanner DDL as view", View.asSingleton());
    jsonRecords.apply("Write events to Cloud Spanner", new SpannerTransactionWriter(sourceConfig, ddlView, shadowTablePrefix, "oracle"));
    PipelineResult testResult = testPipeline.run();
    testResult.waitUntilFinish();
}

Example 32 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class SpannerStreamingWriteIntegrationTest method createTablesForTest.

private void createTablesForTest() throws Exception {
    Ddl ddl = Ddl.builder().createTable("Table1").column("id").int64().endColumn().column("data").int64().endColumn().primaryKey().asc("id").end().endTable().createTable("Table1_interleaved").column("id").int64().endColumn().column("id2").int64().endColumn().column("data2").int64().endColumn().primaryKey().asc("id").desc("id2").end().interleaveInParent("Table1").endTable().build();
    spannerServer.createDatabase(testDb, ddl.statements());
}

Example 33 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ChangeEventConvertorTest method canConvertChangeEventToPrimaryKey.

@Test
public void canConvertChangeEventToPrimaryKey() throws Exception {
    Ddl ddl = getTestDdl();
    JSONObject changeEvent = getTestChangeEvent("Users2");
    JsonNode ce = parseChangeEvent(changeEvent.toString());
    Key key = ChangeEventConvertor.changeEventToPrimaryKey(ddl, ce);
    Iterable<Object> keyParts = key.getParts();
    ArrayList<Object> expectedKeyParts = new ArrayList<>();
    expectedKeyParts.add("A");
    expectedKeyParts.add("B");
    expectedKeyParts.add(Long.valueOf(10));
    expectedKeyParts.add(Boolean.valueOf(true));
    expectedKeyParts.add(Boolean.valueOf(true));
    expectedKeyParts.add(Long.valueOf(2344));
    expectedKeyParts.add(Double.valueOf(2344.34));
    expectedKeyParts.add("testtest");
    expectedKeyParts.add(ByteArray.copyFrom("asdf233sf"));
    expectedKeyParts.add(Timestamp.of(java.sql.Timestamp.valueOf("2020-12-30 4:12:12")));
    expectedKeyParts.add(Timestamp.of(java.sql.Timestamp.valueOf("2020-12-30 4:12:12.1")));
    expectedKeyParts.add(Date.parseDate("2020-12-30"));
    expectedKeyParts.add(Date.parseDate("2020-12-30"));
    assertThat(keyParts, is(expectedKeyParts));
}

Example 34 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ChangeEventConvertorTest method cannotConvertChangeEventWithMissingKeyColToPrimaryKey.

@Test(expected = ChangeEventConvertorException.class)
public void cannotConvertChangeEventWithMissingKeyColToPrimaryKey() throws Exception {
    Ddl ddl = getTestDdl();
    JSONObject changeEvent = getTestChangeEvent("Users2");
    changeEvent.remove("last_name");
    JsonNode ce = parseChangeEvent(changeEvent.toString());
    Key key = ChangeEventConvertor.changeEventToPrimaryKey(ddl, ce);
// Expect an exception since the event is missing a primary key
}

Example 35 with Ddl

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class ChangeEventConvertorTest method cannotConvertChangeEventWithInvalidTimestampToMutation.

@Test(expected = ChangeEventConvertorException.class)
public void cannotConvertChangeEventWithInvalidTimestampToMutation() throws Exception {
    Ddl ddl = getTestDdl();
    JSONObject changeEvent = getTestChangeEvent("Users");
    changeEvent.put("timestamp_field", "2020-12-asdf");
    JsonNode ce = parseChangeEvent(changeEvent.toString());
    Mutation mutation = ChangeEventConvertor.changeEventToMutation(ddl, ce);
}