Examples with PubSubToElasticsearchOptions com.google.cloud.teleport.v2.elasticsearch.options.PubSubToElasticsearchOptions used on opensource projects

Example 1 with PubSubToElasticsearchOptions

use of com.google.cloud.teleport.v2.elasticsearch.options.PubSubToElasticsearchOptions in project DataflowTemplates by GoogleCloudPlatform.

the class PubSubToElasticsearch method main.

/**
 * Main entry point for executing the pipeline.
 *
 * @param args The command-line arguments to the pipeline.
 */
public static void main(String[] args) {
    // Parse the user options passed from the command-line.
    PubSubToElasticsearchOptions pubSubToElasticsearchOptions = PipelineOptionsFactory.fromArgs(args).withValidation().as(PubSubToElasticsearchOptions.class);
    pubSubToElasticsearchOptions.setIndex(new ElasticsearchIndex(pubSubToElasticsearchOptions.getDataset(), pubSubToElasticsearchOptions.getNamespace()).getIndex());
    run(pubSubToElasticsearchOptions);
}

Example 2 with PubSubToElasticsearchOptions

use of com.google.cloud.teleport.v2.elasticsearch.options.PubSubToElasticsearchOptions in project DataflowTemplates by GoogleCloudPlatform.

the class EventMetadataBuilderTest method testEventMetadataAppendFailed.

@Test
public void testEventMetadataAppendFailed() throws IOException {
    exceptionRule.expect(IllegalStateException.class);
    PubSubToElasticsearchOptions options = TestPipeline.testingPipelineOptions().as(PubSubToElasticsearchOptions.class);
    options.setErrorOutputTopic("projects/test/topics/test-error-topic");
    options.setApiKey("key");
    options.setDataset(Dataset.AUDIT);
    options.setNamespace("test-namespace");
    String inputMessageInvalid = readInputMessage(INPUT_MESSAGE_INVALID_FILE_PATH);
    EventMetadataBuilder eventMetadataBuilder = EventMetadataBuilder.build(inputMessageInvalid, options);
    JsonNode enrichedMessageAsJson = eventMetadataBuilder.getEnrichedMessageAsJsonNode();
    // if elasticsearchTemplateVersion is not set, 1.0.0 is the default value
    Assert.assertEquals("1.0.0", enrichedMessageAsJson.get("agent").get("version").textValue());
    Assert.assertEquals(enrichedMessageAsJson.get("data_stream").get("dataset").textValue(), Dataset.AUDIT.getKeyWithPrefix());
}

Example 3 with PubSubToElasticsearchOptions

use of com.google.cloud.teleport.v2.elasticsearch.options.PubSubToElasticsearchOptions in project DataflowTemplates by GoogleCloudPlatform.

the class EventMetadataBuilderTest method testEventMetadata.

@Test
public void testEventMetadata() throws IOException {
    PubSubToElasticsearchOptions options = TestPipeline.testingPipelineOptions().as(PubSubToElasticsearchOptions.class);
    options.setErrorOutputTopic("projects/test/topics/test-error-topic");
    options.setApiKey("key");
    options.setDataset(Dataset.AUDIT);
    options.setNamespace("test-namespace");
    options.setElasticsearchTemplateVersion("999.999.999");
    String inputMessage = readInputMessage(INPUT_MESSAGE_FILE_PATH);
    EventMetadataBuilder eventMetadataBuilder = EventMetadataBuilder.build(inputMessage, options);
    JsonNode enrichedMessageAsJson = eventMetadataBuilder.getEnrichedMessageAsJsonNode();
    String enrichedMessageAsString = eventMetadataBuilder.getEnrichedMessageAsString();
    Assert.assertTrue(StringUtils.isNotBlank(enrichedMessageAsString));
    Assert.assertEquals(inputMessage, enrichedMessageAsJson.get("message").textValue());
    Assert.assertEquals("999.999.999", enrichedMessageAsJson.get("agent").get("version").textValue());
    Assert.assertEquals(Dataset.AUDIT.getKeyWithPrefix(), enrichedMessageAsJson.get("data_stream").get("dataset").textValue());
    Assert.assertEquals("test-namespace", enrichedMessageAsJson.get("data_stream").get("namespace").textValue());
    Assert.assertEquals(Dataset.AUDIT.getKeyWithPrefix(), enrichedMessageAsJson.get("service").get("type").textValue());
    Assert.assertEquals("2021-07-14T10:35:17.528142Z", enrichedMessageAsJson.get("@timestamp").textValue());
}

Example 4 with PubSubToElasticsearchOptions

use of com.google.cloud.teleport.v2.elasticsearch.options.PubSubToElasticsearchOptions in project DataflowTemplates by GoogleCloudPlatform.

the class EventMetadataBuilderTest method testEventMetadataAppend.

@Test
public void testEventMetadataAppend() throws IOException {
    PubSubToElasticsearchOptions options = TestPipeline.testingPipelineOptions().as(PubSubToElasticsearchOptions.class);
    options.setErrorOutputTopic("projects/test/topics/test-error-topic");
    options.setApiKey("key");
    options.setDataset(Dataset.AUDIT);
    options.setNamespace("test-namespace");
    String inputMessage = readInputMessage(INPUT_MESSAGE_FILE_PATH);
    EventMetadataBuilder eventMetadataBuilder = EventMetadataBuilder.build(inputMessage, options);
    JsonNode enrichedMessageAsJson = eventMetadataBuilder.getEnrichedMessageAsJsonNode();
    // if elasticsearchTemplateVersion is not set, 1.0.0 is the default value
    Assert.assertEquals("1.0.0", enrichedMessageAsJson.get("agent").get("version").textValue());
    Assert.assertEquals(enrichedMessageAsJson.get("data_stream").get("dataset").textValue(), Dataset.AUDIT.getKeyWithPrefix());
}

Example 5 with PubSubToElasticsearchOptions

use of com.google.cloud.teleport.v2.elasticsearch.options.PubSubToElasticsearchOptions in project DataflowTemplates by GoogleCloudPlatform.

the class PubSubToElasticsearchTest method testPubSubToElasticsearchNoUdfE2E.

/**
 * Tests the {@link PubSubToElasticsearch} pipeline end-to-end with no UDF supplied.
 */
@Test
public void testPubSubToElasticsearchNoUdfE2E() {
    CoderRegistry coderRegistry = pipeline.getCoderRegistry();
    coderRegistry.registerCoderForType(PubSubToElasticsearch.FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), PubSubToElasticsearch.FAILSAFE_ELEMENT_CODER);
    coderRegistry.registerCoderForType(PubSubToElasticsearch.CODER.getEncodedTypeDescriptor(), PubSubToElasticsearch.CODER);
    PubSubToElasticsearchOptions options = TestPipeline.testingPipelineOptions().as(PubSubToElasticsearchOptions.class);
    options.setErrorOutputTopic("projects/test/topics/test-error-topic");
    options.setJavascriptTextTransformFunctionName(null);
    options.setJavascriptTextTransformGcsPath(null);
    options.setApiKey("key");
    PCollectionTuple pc = pipeline.apply(Create.of(goodTestMessages.get(0))).apply(PubSubMessageToJsonDocument.newBuilder().setJavascriptTextTransformFunctionName(options.getJavascriptTextTransformFunctionName()).setJavascriptTextTransformGcsPath(options.getJavascriptTextTransformGcsPath()).build());
    PAssert.that(pc.get(PubSubToElasticsearch.TRANSFORM_OUT)).satisfies(collection -> {
        FailsafeElement<PubsubMessage, String> element = collection.iterator().next();
        assertThat(element.getOriginalPayload().getPayload(), is(equalTo(goodTestMessages.get(0).getPayload())));
        return null;
    });
    // Execute pipeline
    pipeline.run(options);
}