Examples with ValidateInputFiles com.google.cloud.teleport.spanner.ImportTransform.ValidateInputFiles used on opensource projects

Example 1 with ValidateInputFiles

use of com.google.cloud.teleport.spanner.ImportTransform.ValidateInputFiles in project DataflowTemplates by GoogleCloudPlatform.

the class ImportTransformTest method validateInvalidInputFiles.

@Test(expected = PipelineExecutionException.class)
public void validateInvalidInputFiles() throws Exception {
    Path f1 = Files.createTempFile("table1-file", "1");
    TableManifest.Builder builder = TableManifest.newBuilder();
    builder.addFilesBuilder().setName(f1.getFileName().toString()).setMd5("invalid checksum");
    TableManifest manifest1 = builder.build();
    ValueProvider<String> importDirectory = ValueProvider.StaticValueProvider.of(f1.getParent().toString());
    // Execute the transform.
    pipeline.apply("Create", Create.of(ImmutableMap.of("table1", manifest1))).apply(ParDo.of(new ValidateInputFiles(importDirectory)));
    pipeline.run();
// Pipeline should fail with an exception.
}

Example 2 with ValidateInputFiles

use of com.google.cloud.teleport.spanner.ImportTransform.ValidateInputFiles in project DataflowTemplates by GoogleCloudPlatform.

the class ImportTransformTest method validateInputFiles.

@Test
public void validateInputFiles() throws Exception {
    Path f1 = Files.createTempFile("table1-file", "1");
    Path f2 = Files.createTempFile("table1-file", "2");
    Path f3 = Files.createTempFile("table2-file", "1");
    // Create the expected manifest string.
    TableManifest.Builder builder = TableManifest.newBuilder();
    builder.addFilesBuilder().setName(f1.getFileName().toString()).setMd5("1B2M2Y8AsgTpgAmY7PhCfg==");
    builder.addFilesBuilder().setName(f2.getFileName().toString()).setMd5("1B2M2Y8AsgTpgAmY7PhCfg==");
    TableManifest manifest1 = builder.build();
    builder = TableManifest.newBuilder();
    builder.addFilesBuilder().setName(f3.getFileName().toString()).setMd5("1B2M2Y8AsgTpgAmY7PhCfg==");
    TableManifest manifest2 = builder.build();
    final Map<String, TableManifest> tablesAndManifests = ImmutableMap.of("table1", manifest1, "table2", manifest2);
    ValueProvider<String> importDirectory = ValueProvider.StaticValueProvider.of(f1.getParent().toString());
    // Execute the transform.
    PCollection<KV<String, String>> tableAndFiles = pipeline.apply("Create", Create.of(tablesAndManifests)).apply(ParDo.of(new ValidateInputFiles(importDirectory)));
    PAssert.that(tableAndFiles).containsInAnyOrder(KV.of("table1", f1.toString()), KV.of("table1", f2.toString()), KV.of("table2", f3.toString()));
    pipeline.run();
}