Search in sources :

Example 1 with GCSToSplunkOptions

use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.

the class GCSToSplunkTest method testGCSToSplunkReadHeaders.

@Test
public void testGCSToSplunkReadHeaders() {
    // Arrange
    String stringifiedJsonRecord = "{\"id\":\"008\",\"state\":\"CA\",\"price\":\"26.23\"}";
    SplunkEvent expectedSplunkEvent = SplunkEvent.newBuilder().withEvent(stringifiedJsonRecord).create();
    CoderRegistry coderRegistry = pipeline.getCoderRegistry();
    coderRegistry.registerCoderForClass(SplunkEvent.class, SplunkEventCoder.of());
    coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
    GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
    options.setContainsHeaders(true);
    options.setInputFileSpec(HEADER_CSV_FILE_PATH);
    // Act
    PCollectionTuple readCsvOut = pipeline.apply("Read CSV", readFromCsv(options));
    PCollectionTuple transformedLines = readCsvOut.apply("Convert to JSON", convertToFailsafeAndMaybeApplyUdf(options));
    PCollectionTuple splunkEventTuple = transformedLines.get(UDF_OUT).apply("Convert to Splunk Event", convertToSplunkEvent());
    // Assert
    PAssert.that(transformedLines.get(UDF_OUT)).satisfies(collection -> {
        FailsafeElement element = collection.iterator().next();
        assertThat(element.getPayload()).isEqualTo(stringifiedJsonRecord);
        return null;
    });
    PAssert.that(transformedLines.get(UDF_ERROR_OUT)).empty();
    PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_OUT)).containsInAnyOrder(expectedSplunkEvent);
    PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_ERROR_OUT)).empty();
    // Execute pipeline
    pipeline.run();
}
Also used : GCSToSplunk.convertToSplunkEvent(com.google.cloud.teleport.v2.templates.GCSToSplunk.convertToSplunkEvent) SplunkEvent(org.apache.beam.sdk.io.splunk.SplunkEvent) CoderRegistry(org.apache.beam.sdk.coders.CoderRegistry) PCollectionTuple(org.apache.beam.sdk.values.PCollectionTuple) GCSToSplunkOptions(com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions) GCSToSplunk.flattenErrorsAndConvertToString(com.google.cloud.teleport.v2.templates.GCSToSplunk.flattenErrorsAndConvertToString) FailsafeElement(com.google.cloud.teleport.v2.values.FailsafeElement) Test(org.junit.Test)

Example 2 with GCSToSplunkOptions

use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.

the class GCSToSplunkTest method testGCSToSplunkReadUdf.

@Test
public void testGCSToSplunkReadUdf() {
    // Arrange
    String stringifiedJsonRecord = "{\"id\":\"007\",\"state\":\"CA\",\"price\":26.23}";
    SplunkEvent expectedSplunkEvent = SplunkEvent.newBuilder().withEvent(stringifiedJsonRecord).create();
    CoderRegistry coderRegistry = pipeline.getCoderRegistry();
    coderRegistry.registerCoderForClass(SplunkEvent.class, SplunkEventCoder.of());
    coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
    GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
    options.setJavascriptTextTransformGcsPath(TRANSFORM_FILE_PATH);
    options.setJavascriptTextTransformFunctionName("transform");
    options.setContainsHeaders(false);
    options.setInputFileSpec(NO_HEADER_CSV_FILE_PATH);
    // Act
    PCollectionTuple readCsvOut = pipeline.apply("Read CSV", readFromCsv(options));
    PCollectionTuple transformedLines = readCsvOut.apply("Convert to JSON", convertToFailsafeAndMaybeApplyUdf(options));
    PCollectionTuple splunkEventTuple = transformedLines.get(UDF_OUT).apply("Convert to Splunk Event", convertToSplunkEvent());
    // Assert
    PAssert.that(transformedLines.get(UDF_OUT)).satisfies(collection -> {
        FailsafeElement element = collection.iterator().next();
        assertThat(element.getPayload()).isEqualTo(stringifiedJsonRecord);
        return null;
    });
    PAssert.that(transformedLines.get(UDF_ERROR_OUT)).empty();
    PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_OUT)).containsInAnyOrder(expectedSplunkEvent);
    PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_ERROR_OUT)).empty();
    // Execute pipeline
    pipeline.run();
}
Also used : GCSToSplunk.convertToSplunkEvent(com.google.cloud.teleport.v2.templates.GCSToSplunk.convertToSplunkEvent) SplunkEvent(org.apache.beam.sdk.io.splunk.SplunkEvent) CoderRegistry(org.apache.beam.sdk.coders.CoderRegistry) PCollectionTuple(org.apache.beam.sdk.values.PCollectionTuple) GCSToSplunkOptions(com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions) GCSToSplunk.flattenErrorsAndConvertToString(com.google.cloud.teleport.v2.templates.GCSToSplunk.flattenErrorsAndConvertToString) FailsafeElement(com.google.cloud.teleport.v2.values.FailsafeElement) Test(org.junit.Test)

Example 3 with GCSToSplunkOptions

use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.

the class GCSToSplunkTest method testGCSToSplunkReadJsonSchema.

@Test
public void testGCSToSplunkReadJsonSchema() {
    // Arrange
    String stringifiedJsonRecord = "{\"id\":\"007\",\"state\":\"CA\",\"price\":26.23}";
    SplunkEvent expectedSplunkEvent = SplunkEvent.newBuilder().withEvent(stringifiedJsonRecord).create();
    CoderRegistry coderRegistry = pipeline.getCoderRegistry();
    coderRegistry.registerCoderForClass(SplunkEvent.class, SplunkEventCoder.of());
    coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
    GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
    options.setJsonSchemaPath(JSON_SCHEMA_FILE_PATH);
    options.setContainsHeaders(false);
    options.setInputFileSpec(NO_HEADER_CSV_FILE_PATH);
    // Act
    PCollectionTuple readCsvOut = pipeline.apply("Read CSV", readFromCsv(options));
    PCollectionTuple transformedLines = readCsvOut.apply("Convert to JSON", convertToFailsafeAndMaybeApplyUdf(options));
    PCollectionTuple splunkEventTuple = transformedLines.get(UDF_OUT).apply("Convert to Splunk Event", convertToSplunkEvent());
    // Assert
    PAssert.that(transformedLines.get(UDF_OUT)).satisfies(collection -> {
        FailsafeElement element = collection.iterator().next();
        assertThat(element.getPayload()).isEqualTo(stringifiedJsonRecord);
        return null;
    });
    PAssert.that(transformedLines.get(UDF_ERROR_OUT)).empty();
    PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_OUT)).containsInAnyOrder(expectedSplunkEvent);
    PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_ERROR_OUT)).empty();
    // Execute pipeline
    pipeline.run();
}
Also used : GCSToSplunk.convertToSplunkEvent(com.google.cloud.teleport.v2.templates.GCSToSplunk.convertToSplunkEvent) SplunkEvent(org.apache.beam.sdk.io.splunk.SplunkEvent) CoderRegistry(org.apache.beam.sdk.coders.CoderRegistry) PCollectionTuple(org.apache.beam.sdk.values.PCollectionTuple) GCSToSplunkOptions(com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions) GCSToSplunk.flattenErrorsAndConvertToString(com.google.cloud.teleport.v2.templates.GCSToSplunk.flattenErrorsAndConvertToString) FailsafeElement(com.google.cloud.teleport.v2.values.FailsafeElement) Test(org.junit.Test)

Example 4 with GCSToSplunkOptions

use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.

the class GCSToSplunkTest method testGCSToSplunkWriteErrorsToFolder.

@Test
public void testGCSToSplunkWriteErrorsToFolder() throws IOException {
    // Arrange
    String stringifiedSplunkError = "Payload: test-payload. Error Message: test-message. Splunk write status code: 123.";
    PCollection<String> stringifiedErrorCollection = pipeline.apply("Add Stringified Errors", Create.of(stringifiedSplunkError).withCoder(StringUtf8Coder.of()));
    PCollectionTuple stringifiedErrorTuple = PCollectionTuple.of(COMBINED_ERRORS, stringifiedErrorCollection);
    GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
    options.setInvalidOutputPath(tmpFolder.getRoot().getAbsolutePath() + "errors.txt");
    // Act
    stringifiedErrorTuple.apply("Output Errors To GCS", writeErrorsToGCS(options));
    // Execute pipeline
    pipeline.run();
    // Assert
    File file = new File(tmpFolder.getRoot().getAbsolutePath() + "errors.txt-00000-of-00001");
    String fileContents = Files.toString(file, Charsets.UTF_8);
    assertThat(fileContents).contains(stringifiedSplunkError);
}
Also used : PCollectionTuple(org.apache.beam.sdk.values.PCollectionTuple) GCSToSplunkOptions(com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions) GCSToSplunk.flattenErrorsAndConvertToString(com.google.cloud.teleport.v2.templates.GCSToSplunk.flattenErrorsAndConvertToString) File(java.io.File) Test(org.junit.Test)

Aggregations

GCSToSplunkOptions (com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions)4 GCSToSplunk.flattenErrorsAndConvertToString (com.google.cloud.teleport.v2.templates.GCSToSplunk.flattenErrorsAndConvertToString)4 PCollectionTuple (org.apache.beam.sdk.values.PCollectionTuple)4 Test (org.junit.Test)4 GCSToSplunk.convertToSplunkEvent (com.google.cloud.teleport.v2.templates.GCSToSplunk.convertToSplunkEvent)3 FailsafeElement (com.google.cloud.teleport.v2.values.FailsafeElement)3 CoderRegistry (org.apache.beam.sdk.coders.CoderRegistry)3 SplunkEvent (org.apache.beam.sdk.io.splunk.SplunkEvent)3 File (java.io.File)1