Examples with Options com.google.cloud.teleport.v2.transforms.DeleteBigQueryDataFn.Options used on opensource projects

Example 21 with Options

use of com.google.cloud.teleport.v2.transforms.DeleteBigQueryDataFn.Options in project DataflowTemplates by GoogleCloudPlatform.

the class EventMetadataBuilderTest method testEventMetadata.

@Test
public void testEventMetadata() throws IOException {
    PubSubToElasticsearchOptions options = TestPipeline.testingPipelineOptions().as(PubSubToElasticsearchOptions.class);
    options.setErrorOutputTopic("projects/test/topics/test-error-topic");
    options.setApiKey("key");
    options.setDataset(Dataset.AUDIT);
    options.setNamespace("test-namespace");
    options.setElasticsearchTemplateVersion("999.999.999");
    String inputMessage = readInputMessage(INPUT_MESSAGE_FILE_PATH);
    EventMetadataBuilder eventMetadataBuilder = EventMetadataBuilder.build(inputMessage, options);
    JsonNode enrichedMessageAsJson = eventMetadataBuilder.getEnrichedMessageAsJsonNode();
    String enrichedMessageAsString = eventMetadataBuilder.getEnrichedMessageAsString();
    Assert.assertTrue(StringUtils.isNotBlank(enrichedMessageAsString));
    Assert.assertEquals(inputMessage, enrichedMessageAsJson.get("message").textValue());
    Assert.assertEquals("999.999.999", enrichedMessageAsJson.get("agent").get("version").textValue());
    Assert.assertEquals(Dataset.AUDIT.getKeyWithPrefix(), enrichedMessageAsJson.get("data_stream").get("dataset").textValue());
    Assert.assertEquals("test-namespace", enrichedMessageAsJson.get("data_stream").get("namespace").textValue());
    Assert.assertEquals(Dataset.AUDIT.getKeyWithPrefix(), enrichedMessageAsJson.get("service").get("type").textValue());
    Assert.assertEquals("2021-07-14T10:35:17.528142Z", enrichedMessageAsJson.get("@timestamp").textValue());
}

Example 22 with Options

use of com.google.cloud.teleport.v2.transforms.DeleteBigQueryDataFn.Options in project DataflowTemplates by GoogleCloudPlatform.

the class WriteToElasticsearchTest method testNullConnectionInformation.

/**
 * Tests {@link WriteToElasticsearch} throws an exception if a null ConnectionInformation is
 * provided.
 */
@Test
public void testNullConnectionInformation() {
    exceptionRule.expect(IllegalArgumentException.class);
    ElasticsearchWriteOptions options = PipelineOptionsFactory.create().as(ElasticsearchWriteOptions.class);
    options.setConnectionUrl(null);
    options.setApiKey("key");
    pipeline.apply("CreateInput", Create.of("test")).apply(WriteToElasticsearch.newBuilder().setOptions(options).build());
    pipeline.run();
}

Example 23 with Options

use of com.google.cloud.teleport.v2.transforms.DeleteBigQueryDataFn.Options in project DataflowTemplates by GoogleCloudPlatform.

the class WriteToElasticsearchTest method testNullType.

/**
 * Tests {@link WriteToElasticsearch} throws an exception if a null index is provided.
 */
@Test
public void testNullType() {
    exceptionRule.expect(IllegalArgumentException.class);
    ElasticsearchWriteOptions options = PipelineOptionsFactory.create().as(ElasticsearchWriteOptions.class);
    options.setConnectionUrl("https://host.domain");
    options.setApiKey("key");
    pipeline.apply("CreateInput", Create.of("test")).apply(WriteToElasticsearch.newBuilder().setOptions(options).build());
    pipeline.run();
}

Example 24 with Options

use of com.google.cloud.teleport.v2.transforms.DeleteBigQueryDataFn.Options in project DataflowTemplates by GoogleCloudPlatform.

the class WriteToElasticsearchTest method testElasticsearchWriteOptionsRetryConfigMaxAttempts.

/**
 * Tests that {@link WriteToElasticsearch} throws an exception if {@link
 * org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO.RetryConfiguration} are invalid.
 */
@Test
public void testElasticsearchWriteOptionsRetryConfigMaxAttempts() {
    exceptionRule.expect(IllegalArgumentException.class);
    ElasticsearchWriteOptions options = PipelineOptionsFactory.create().as(ElasticsearchWriteOptions.class);
    options.setConnectionUrl("https://host.domain");
    options.setApiKey("key");
    options.setMaxRetryDuration(500L);
    options.setMaxRetryAttempts(null);
    pipeline.apply("CreateInput", Create.of("test")).apply("TestWriteToElasticsearchBadMaxAttempts", WriteToElasticsearch.newBuilder().setOptions(options).build());
    pipeline.run();
}

Example 25 with Options

use of com.google.cloud.teleport.v2.transforms.DeleteBigQueryDataFn.Options in project DataflowTemplates by GoogleCloudPlatform.

the class DataStreamToSpanner method getSourceType.

private static String getSourceType(Options options) {
    if (options.getDatastreamSourceType() != null) {
        return options.getDatastreamSourceType();
    }
    if (options.getStreamName() == null) {
        throw new IllegalArgumentException("Stream name cannot be empty. ");
    }
    GcpOptions gcpOptions = options.as(GcpOptions.class);
    DataStreamClient datastreamClient;
    SourceConfig sourceConfig;
    try {
        datastreamClient = new DataStreamClient(gcpOptions.getGcpCredential());
        sourceConfig = datastreamClient.getSourceConnectionProfile(options.getStreamName());
    } catch (IOException e) {
        LOG.error("IOException Occurred: DataStreamClient failed initialization.");
        throw new IllegalArgumentException("Unable to initialize DatastreamClient: " + e);
    }
    if (sourceConfig.getMysqlSourceConfig() != null) {
        return DatastreamConstants.MYSQL_SOURCE_TYPE;
    } else if (sourceConfig.getOracleSourceConfig() != null) {
        return DatastreamConstants.ORACLE_SOURCE_TYPE;
    }
    LOG.error("Source Connection Profile Type Not Supported");
    throw new IllegalArgumentException("Unsupported source connection profile type in Datastream");
}