Examples with KafkaToGCSOptions com.google.cloud.teleport.v2.options.KafkaToGCSOptions used on opensource projects

Example 1 with KafkaToGCSOptions

use of com.google.cloud.teleport.v2.options.KafkaToGCSOptions in project DataflowTemplates by GoogleCloudPlatform.

the class KafkaToGCSTest method testFileFormatFactoryInvalid.

/**
 * Test whether {@link FileFormatFactory} maps the output file format to the transform to be
 * carried out. And throws illegal argument exception if invalid file format is passed.
 */
@Test
@Category(NeedsRunner.class)
public void testFileFormatFactoryInvalid() {
    // Create the test input.
    final String key = "Name";
    final String value = "Generic";
    final KV<String, String> message = KV.of(key, value);
    final String outputDirectory = "gs://bucket_name/path/to/output-location";
    final String outputFileFormat = "json".toUpperCase();
    final String outputFilenamePrefix = "output";
    final Integer numShards = 1;
    final String tempOutputDirectory = "gs://bucket_name/path/to/temp-location";
    KafkaToGCSOptions options = PipelineOptionsFactory.create().as(KafkaToGCSOptions.class);
    options.setOutputFileFormat(outputFileFormat);
    options.setOutputDirectory(outputDirectory);
    options.setOutputFilenamePrefix(outputFilenamePrefix);
    options.setNumShards(numShards);
    options.setTempLocation(tempOutputDirectory);
    exception.expect(IllegalArgumentException.class);
    PCollection<KV<String, String>> records = pipeline.apply("CreateInput", Create.of(message).withCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())));
    records.apply("WriteToGCS", FileFormatFactory.newBuilder().setOptions(options).build());
    // Run the pipeline.
    pipeline.run();
}

Example 2 with KafkaToGCSOptions

use of com.google.cloud.teleport.v2.options.KafkaToGCSOptions in project DataflowTemplates by GoogleCloudPlatform.

the class KafkaToGCS method main.

/**
 * Main entry point for pipeline execution.
 *
 * @param args Command line arguments to the pipeline.
 */
public static void main(String[] args) {
    KafkaToGCSOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(KafkaToGCSOptions.class);
    run(options);
}