Examples with Ddl com.google.cloud.teleport.spanner.ddl.Ddl used on opensource projects

Example 31 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class TextImportTransformTest method readImportManifestGeneratedColumn.

@Test
public void readImportManifestGeneratedColumn() throws Exception {
    Path f31 = Files.createTempFile("table3-file", "1");
    Path manifestFile = Files.createTempFile("import-manifest", ".json");
    Charset charset = Charset.forName("UTF-8");
    try (BufferedWriter writer = Files.newBufferedWriter(manifestFile, charset)) {
        String jsonString = String.format("{\"tables\": [" + "{\"table_name\": \"table3\"," + "\"file_patterns\": [\"%s\"]," + "\"columns\": [{\"column_name\": \"int_col\", \"type_name\": \"INT64\"}]}" + "]}", f31.toString());
        writer.write(jsonString, 0, jsonString.length());
    } catch (IOException e) {
        e.printStackTrace();
    }
    ValueProvider<String> importManifest = ValueProvider.StaticValueProvider.of(manifestFile.toString());
    PCollectionView<Ddl> ddlView = pipeline.apply("ddl", Create.of(getTestDdl())).apply(View.asSingleton());
    PCollection<KV<String, String>> tableAndFiles = pipeline.apply("Read manifest file", new ReadImportManifest(importManifest)).apply("Resolve data files", new ResolveDataFiles(importManifest, ddlView));
    pipeline.run();
}

Example 32 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class TextImportTransformTest method readImportManifestUtfWithBOM.

@Test
public void readImportManifestUtfWithBOM() throws Exception {
    Path f11 = Files.createTempFile("table1-file", "1");
    String tempDir = f11.getParent().toString();
    Path manifestFile = Files.createTempFile("import-manifest", ".json");
    Charset charset = Charset.forName("UTF-8");
    try (BufferedWriter writer = Files.newBufferedWriter(manifestFile, charset)) {
        String jsonString = String.format("\uFEFF{\"tables\": [" + "{\"table_name\": \"table1\"," + "\"file_patterns\":[\"%s\"]}" + "]}", f11.toString());
        writer.write(jsonString, 0, jsonString.length());
    } catch (IOException e) {
        e.printStackTrace();
    }
    ValueProvider<String> importManifest = ValueProvider.StaticValueProvider.of(manifestFile.toString());
    PCollectionView<Ddl> ddlView = pipeline.apply("ddl", Create.of(getTestDdl())).apply(View.asSingleton());
    PCollection<KV<String, String>> tableAndFiles = pipeline.apply("Read manifest file", new ReadImportManifest(importManifest)).apply("Resolve data files", new ResolveDataFiles(importManifest, ddlView));
    PAssert.that(tableAndFiles).containsInAnyOrder(KV.of("table1", f11.toString()));
    pipeline.run();
}

Example 33 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class TextImportTransformTest method readImportManifestColumnListMustBeProvidedForGeneratedColumn.

@Test
public void readImportManifestColumnListMustBeProvidedForGeneratedColumn() throws Exception {
    Path f31 = Files.createTempFile("table3-file", "1");
    Path manifestFile = Files.createTempFile("import-manifest", ".json");
    Charset charset = Charset.forName("UTF-8");
    try (BufferedWriter writer = Files.newBufferedWriter(manifestFile, charset)) {
        String jsonString = String.format("{\"tables\": [" + "{\"table_name\": \"table3\"," + "\"file_patterns\": [\"%s\"]}" + "]}", f31.toString());
        writer.write(jsonString, 0, jsonString.length());
    } catch (IOException e) {
        e.printStackTrace();
    }
    ValueProvider<String> importManifest = ValueProvider.StaticValueProvider.of(manifestFile.toString());
    PCollectionView<Ddl> ddlView = pipeline.apply("ddl", Create.of(getTestDdl())).apply(View.asSingleton());
    PCollection<KV<String, String>> tableAndFiles = pipeline.apply("Read manifest file", new ReadImportManifest(importManifest)).apply("Resolve data files", new ResolveDataFiles(importManifest, ddlView));
    try {
        pipeline.run();
    } catch (PipelineExecutionException e) {
        assertThat(e.getMessage(), equalTo("java.lang.RuntimeException: DB table table3 has one or more generated columns. An" + " explicit column list that excludes the generated columns must be provided in" + " the manifest."));
    }
}

Example 34 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class SpannerToTextTest method runPgExportWithTsTest.

@Test
public void runPgExportWithTsTest() throws Exception {
    Ddl ddl = Ddl.builder(Dialect.POSTGRESQL).createTable(tableName).column("first_name").pgVarchar().max().endColumn().column("last_name").pgVarchar().size(5).endColumn().column("age").pgInt8().endColumn().primaryKey().asc("first_name").desc("last_name").end().endTable().build();
    /* Create initial table and populate
     * numBatches = 100
     */
    createAndPopulate(sourceDb, ddl, 100);
    // Export the database and note the timestamp ts1
    exportDbAtTime(sourceDb, destDbPrefix + chkpt1, chkpt1, "", exportPipeline1, tmpDir);
    // Save the timestamp directly after the export
    String chkPt1Ts = getCurrentTimestamp(Dialect.POSTGRESQL);
    // Add more records to the table, export the database and note the timestamp ts3
    spannerServer.populateRandomData(sourceDb, ddl, 100);
    // Export the table from the database using the saved timestamp
    exportDbAtTime(sourceDb, destDbPrefix + chkpt2, chkpt2, chkPt1Ts, exportPipeline2, tmpDir);
    File folder = new File(tmpDir + "/");
    // Store the contents of directory containing the exported CSVs into a List
    File[] files = folder.listFiles();
    List<String> oldData = readDbData(files, chkpt1);
    List<String> expectedOldData = readDbData(files, chkpt2);
    // Sort statements
    Collections.sort(oldData);
    Collections.sort(expectedOldData);
    assertEquals(oldData, expectedOldData);
}

Example 35 with Ddl

use of com.google.cloud.teleport.spanner.ddl.Ddl in project DataflowTemplates by GoogleCloudPlatform.

the class SpannerToTextTest method runExportWithTsTest.

/* Validates behavior of database export without specifying timestamp
   * and with timestamp specified */
@Test
public void runExportWithTsTest() throws Exception {
    Ddl ddl = Ddl.builder().createTable(tableName).column("first_name").string().max().endColumn().column("last_name").string().size(5).endColumn().column("age").int64().endColumn().primaryKey().asc("first_name").desc("last_name").end().endTable().build();
    /* Create initial table and populate
     * numBatches = 100
     */
    createAndPopulate(sourceDb, ddl, 100);
    // Export the database and note the timestamp ts1
    exportDbAtTime(sourceDb, destDbPrefix + chkpt1, chkpt1, "", exportPipeline1, tmpDir);
    // Save the timestamp directly after the export
    String chkPt1Ts = getCurrentTimestamp(Dialect.GOOGLE_STANDARD_SQL);
    // Add more records to the table, export the database and note the timestamp ts3
    spannerServer.populateRandomData(sourceDb, ddl, 100);
    // Export the table from the database using the saved timestamp
    exportDbAtTime(sourceDb, destDbPrefix + chkpt2, chkpt2, chkPt1Ts, exportPipeline2, tmpDir);
    File folder = new File(tmpDir + "/");
    // Store the contents of directory containing the exported CSVs into a List
    File[] files = folder.listFiles();
    List<String> oldData = readDbData(files, chkpt1);
    List<String> expectedOldData = readDbData(files, chkpt2);
    // Sort statements
    Collections.sort(oldData);
    Collections.sort(expectedOldData);
    assertEquals(oldData, expectedOldData);
}