Examples with Ddl com.google.cloud.teleport.spanner.ddl.Ddl used on opensource projects

Example 36 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class AvroSchemaToDdlConverterTest method pgInvokerRightsView.

@Test
public void pgInvokerRightsView() {
    String avroString = "{" + "  \"type\" : \"record\"," + "  \"name\" : \"Names\"," + "  \"fields\" : []," + "  \"namespace\" : \"spannertest\"," + "  \"googleStorage\" : \"CloudSpanner\"," + "  \"googleFormatVersion\" : \"booleans\"," + "  \"spannerViewSecurity\" : \"INVOKER\"," + "  \"spannerViewQuery\" : \"SELECT first_name, last_name FROM Users\"" + "}";
    Schema schema = new Schema.Parser().parse(avroString);
    AvroSchemaToDdlConverter converter = new AvroSchemaToDdlConverter(Dialect.POSTGRESQL);
    Ddl ddl = converter.toDdl(Collections.singleton(schema));
    assertEquals(ddl.dialect(), Dialect.POSTGRESQL);
    assertThat(ddl.views(), hasSize(1));
    assertThat(ddl.prettyPrint(), equalToCompressingWhiteSpace("CREATE VIEW \"Names\" SQL SECURITY INVOKER AS SELECT first_name, last_name FROM" + " Users"));
}

Example 37 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class AvroSchemaToDdlConverterTest method pgSimple.

@Test
public void pgSimple() {
    String avroString = "{" + "  \"type\" : \"record\"," + "  \"name\" : \"Users\"," + "  \"namespace\" : \"spannertest\"," + "  \"fields\" : [ {" + "    \"name\" : \"id\"," + "    \"type\" : \"long\"," + "    \"sqlType\" : \"bigint\"" + "  }, {" + "    \"name\" : \"first_name\"," + "    \"type\" : [ \"null\", \"string\" ]," + "    \"sqlType\" : \"character varying(10)\"," + "    \"defaultExpression\" : \"'John'\"" + "  }, {" + "    \"name\" : \"last_name\"," + "    \"type\" : [ \"null\", \"string\" ]," + "    \"sqlType\" : \"character varying\"" + "  }, {" + "    \"name\" : \"full_name\"," + "    \"type\" : \"null\"," + "    \"sqlType\" : \"character varying\"," + "    \"notNull\" : \"false\"," + "    \"generationExpression\" : \"CONCAT(first_name, ' ', last_name)\"," + "    \"stored\" : \"true\"" + "  }, {" + "    \"name\" : \"numeric\"," + "    \"type\" : [\"null\", {\"type\":\"bytes\",\"logicalType\":\"decimal\"}]," + "    \"sqlType\" : \"numeric\"" + "  }, {" + "    \"name\" : \"numeric2\"," + "    \"type\" : [\"null\", {\"type\":\"bytes\",\"logicalType\":\"decimal\"," + // Omitting sqlType
    "                \"precision\":147455,\"scale\":16383}]" + "  }, {" + "    \"name\" : \"notNumeric\"," + "    \"type\" : [\"null\", {\"type\":\"bytes\",\"logicalType\":\"decimal\"," + // Omitting sqlType
    "                \"precision\":147455}]" + "  }, {" + "    \"name\" : \"notNumeric2\"," + "    \"type\" : [\"null\", {\"type\":\"bytes\",\"logicalType\":\"decimal\"," + // Omitting sqlType
    "                \"precision\":147455,\"scale\":16384}]" + "  }, {" + "    \"name\":\"numericArr\"," + "    \"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\",{\"type\":\"bytes\"," + "    \"logicalType\":\"decimal\",\"precision\":147455,\"scale\":16383}]}]" + // Omitting sqlType
    "  }, {" + "    \"name\":\"notNumericArr\"," + "    \"type\":[\"null\",{\"type\":\"array\",\"items\":[\"null\",{\"type\":\"bytes\"," + // Omitting sqlType
    "    \"logicalType\":\"decimal\",\"precision\":147455}]}]" + "  }, {" + "    \"name\" : \"bool\"," + "    \"type\" : [ \"null\", \"boolean\" ]," + "    \"sqlType\" : \"boolean\"" + "  }, {" + "    \"name\" : \"float\"," + "    \"type\" : [ \"null\", \"double\" ]," + "    \"sqlType\" : \"double precision\"" + "  }, {" + "    \"name\" : \"bytes\"," + "    \"type\" : [ \"null\", \"bytes\" ]," + "    \"sqlType\" : \"bytea\"" + "  }, {" + "    \"name\" : \"text\"," + "    \"type\" : [ \"null\", \"string\" ]," + "    \"sqlType\" : \"text\"" + "  }, {" + "    \"name\" : \"timestamptz\"," + "    \"type\" : [ \"null\", \"string\" ]," + "    \"sqlType\" : \"timestamp with time zone\"" + "  }, {" + "    \"name\" : \"date\"," + "    \"type\" : [ \"null\", \"string\" ]," + "    \"sqlType\" : \"date\"" + "  }, {" + "    \"name\" : \"varcharArr1\"," + "    \"type\" : [\"null\"," + "               {\"type\":\"array\",\"items\":[\"null\",{\"type\":\"string\"}]}]," + "    \"sqlType\" : \"character varying[]\"" + "  }, {" + "    \"name\" : \"varcharArr2\"," + "    \"type\" : [\"null\"," + "               {\"type\":\"array\",\"items\":[\"null\",{\"type\":\"string\"}]}]" + // Omitting sqlType
    "  } ],  \"googleStorage\" : \"CloudSpanner\",  \"spannerParent\" : \"\", " + " \"googleFormatVersion\" : \"booleans\",  \"spannerPrimaryKey_0\" : \"\\\"id\\\"" + " ASC\",  \"spannerPrimaryKey_1\" : \"\\\"last_name\\\" ASC\",  \"spannerIndex_0\" :" + "   \"CREATE INDEX \\\"UsersByFirstName\\\" ON \\\"Users\\\" (\\\"first_name\\\")\", " + " \"spannerForeignKey_0\" :   \"ALTER TABLE \\\"Users\\\" ADD CONSTRAINT \\\"fk\\\"" + " FOREIGN KEY (\\\"first_name\\\")   REFERENCES \\\"AllowedNames\\\"" + " (\\\"first_name\\\")\",  \"spannerCheckConstraint_0\" :   \"CONSTRAINT \\\"ck\\\"" + " CHECK(\\\"first_name\\\" != \\\"last_name\\\")\"}";
    Schema schema = new Schema.Parser().parse(avroString);
    AvroSchemaToDdlConverter converter = new AvroSchemaToDdlConverter(Dialect.POSTGRESQL);
    Ddl ddl = converter.toDdl(Collections.singleton(schema));
    assertEquals(ddl.dialect(), Dialect.POSTGRESQL);
    assertThat(ddl.allTables(), hasSize(1));
    assertThat(ddl.views(), hasSize(0));
    assertThat(ddl.prettyPrint(), equalToCompressingWhiteSpace("CREATE TABLE \"Users\" (" + " \"id\"              bigint NOT NULL," + " \"first_name\"      character varying(10) DEFAULT 'John'," + " \"last_name\"       character varying," + " \"full_name\"       character varying GENERATED ALWAYS AS" + " (CONCAT(first_name, ' ', last_name)) STORED," + " \"numeric\"         numeric," + " \"numeric2\"        numeric," + " \"notNumeric\"      bytea," + " \"notNumeric2\"     bytea," + " \"numericArr\"         numeric[]," + " \"notNumericArr\"      bytea[]," + " \"bool\" boolean," + " \"float\" double precision," + " \"bytes\" bytea," + " \"text\" text," + " \"timestamptz\" timestamp with time zone," + " \"date\" date," + " \"varcharArr1\"     character varying[]," + " \"varcharArr2\"     character varying[]," + " CONSTRAINT \"ck\" CHECK(\"first_name\" != \"last_name\")," + " PRIMARY KEY (\"id\", \"last_name\")" + " )" + " CREATE INDEX \"UsersByFirstName\" ON \"Users\" (\"first_name\")" + " ALTER TABLE \"Users\" ADD CONSTRAINT \"fk\" FOREIGN KEY (\"first_name\")" + " REFERENCES \"AllowedNames\" (\"first_name\")"));
}

Example 38 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class AvroSchemaToDdlConverterTest method pgEmptySchema.

@Test
public void pgEmptySchema() {
    AvroSchemaToDdlConverter converter = new AvroSchemaToDdlConverter(Dialect.POSTGRESQL);
    Ddl ddl = converter.toDdl(Collections.emptyList());
    assertEquals(ddl.dialect(), Dialect.POSTGRESQL);
    assertThat(ddl.allTables(), empty());
}

Example 39 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class AvroTableFileAsMutationsTest method testAvroToMutationsTransform.

@Test
public void testAvroToMutationsTransform() throws Exception {
    DdlToAvroSchemaConverter converter = new DdlToAvroSchemaConverter("spannertest", "booleans", false);
    Ddl ddl = Ddl.builder().createTable("Users").column("id").int64().notNull().endColumn().column("first_name").string().size(10).endColumn().column("last_name").type(Type.string()).max().endColumn().column("full_name").type(Type.string()).max().generatedAs("CONCAT(first_name, ' ', last_name)").stored().endColumn().primaryKey().asc("id").desc("last_name").end().endTable().build();
    Collection<Schema> result = converter.convert(ddl);
    assertThat(result, hasSize(1));
    Schema usersSchema = result.iterator().next();
    GenericRecord user1 = new GenericData.Record(usersSchema);
    user1.put("id", 123L);
    user1.put("first_name", "John");
    user1.put("last_name", "Smith");
    user1.put("full_name", "John Smith");
    GenericRecord user2 = new GenericData.Record(usersSchema);
    user2.put("id", 456L);
    user2.put("first_name", "Jane");
    user2.put("last_name", "Doe");
    user2.put("full_name", "Jane Doe");
    File file = tmpFolder.newFile("users.avro");
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(usersSchema);
    DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter);
    dataFileWriter.create(usersSchema, file);
    dataFileWriter.append(user1);
    dataFileWriter.append(user2);
    dataFileWriter.close();
    PCollectionView<Ddl> ddlView = p.apply("ddl", Create.of(ddl)).apply(View.asSingleton());
    PCollection<Mutation> mutations = p.apply("files/tables", Create.of(ImmutableMap.of(file.toPath().toString(), "Users"))).apply(new AvroTableFileAsMutations(ddlView));
    PAssert.that(mutations).containsInAnyOrder(Mutation.newInsertOrUpdateBuilder("Users").set("id").to(123L).set("first_name").to("John").set("last_name").to("Smith").build(), Mutation.newInsertOrUpdateBuilder("Users").set("id").to(456L).set("first_name").to("Jane").set("last_name").to("Doe").build());
    p.run();
}

Example 40 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class CopyDbTest method emptyDb.

@Test
public void emptyDb() throws Exception {
    Ddl ddl = Ddl.builder().build();
    createAndPopulate(ddl, 0);
    runTest();
}