Examples with PubsubToJdbcOptions com.google.cloud.teleport.v2.options.PubsubToJdbcOptions used on opensource projects

Example 1 with PubsubToJdbcOptions

use of com.google.cloud.teleport.v2.options.PubsubToJdbcOptions in project DataflowTemplates by GoogleCloudPlatform.

the class PubsubToJdbc method main.

/**
 * Main entry point for pipeline execution.
 *
 * @param args Command line arguments to the pipeline.
 */
public static void main(String[] args) {
    PubsubToJdbcOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(PubsubToJdbcOptions.class);
    run(options);
}

Example 2 with PubsubToJdbcOptions

use of com.google.cloud.teleport.v2.options.PubsubToJdbcOptions in project DataflowTemplates by GoogleCloudPlatform.

the class PubsubToJdbc method run.

/**
 * Runs a pipeline which reads message from Pub/Sub and writes to JdbcIO.
 *
 * @param options The execution options.
 * @return The pipeline result.
 */
public static PipelineResult run(PubsubToJdbcOptions options) {
    // Create the pipeline
    Pipeline pipeline = Pipeline.create(options);
    LOG.info("Starting Pubsub-to-Jdbc Pipeline.");
    /*
     * Steps:
     *  1) Read data from a Pub/Sub subscription
     *  2) Write to Jdbc Table
     *  3) Write errors to deadletter topic
     */
    PCollection<String> pubsubData = pipeline.apply("readFromPubSubSubscription", PubsubIO.readStrings().fromSubscription(options.getInputSubscription()));
    DynamicJdbcIO.DynamicDataSourceConfiguration dataSourceConfiguration = DynamicJdbcIO.DynamicDataSourceConfiguration.create(options.getDriverClassName(), maybeDecrypt(options.getConnectionUrl(), options.getKMSEncryptionKey())).withDriverJars(options.getDriverJars());
    if (options.getUsername() != null) {
        dataSourceConfiguration = dataSourceConfiguration.withUsername(maybeDecrypt(options.getUsername(), options.getKMSEncryptionKey()));
    }
    if (options.getPassword() != null) {
        dataSourceConfiguration = dataSourceConfiguration.withPassword(maybeDecrypt(options.getPassword(), options.getKMSEncryptionKey()));
    }
    if (options.getConnectionProperties() != null) {
        dataSourceConfiguration = dataSourceConfiguration.withConnectionProperties(options.getConnectionProperties());
    }
    PCollection<FailsafeElement<String, String>> errors = pubsubData.apply("writeToJdbc", DynamicJdbcIO.<String>write().withDataSourceConfiguration(dataSourceConfiguration).withStatement(options.getStatement()).withPreparedStatementSetter(new MapJsonStringToQuery(getKeyOrder(options.getStatement())))).setCoder(FAILSAFE_ELEMENT_CODER);
    errors.apply("WriteFailedRecords", ErrorConverters.WriteStringMessageErrorsToPubSub.newBuilder().setErrorRecordsTopic(options.getOutputDeadletterTopic()).build());
    return pipeline.run();
}