use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.
the class GCSToSplunkTest method testGCSToSplunkReadHeaders.
@Test
public void testGCSToSplunkReadHeaders() {
// Arrange
String stringifiedJsonRecord = "{\"id\":\"008\",\"state\":\"CA\",\"price\":\"26.23\"}";
SplunkEvent expectedSplunkEvent = SplunkEvent.newBuilder().withEvent(stringifiedJsonRecord).create();
CoderRegistry coderRegistry = pipeline.getCoderRegistry();
coderRegistry.registerCoderForClass(SplunkEvent.class, SplunkEventCoder.of());
coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
options.setContainsHeaders(true);
options.setInputFileSpec(HEADER_CSV_FILE_PATH);
// Act
PCollectionTuple readCsvOut = pipeline.apply("Read CSV", readFromCsv(options));
PCollectionTuple transformedLines = readCsvOut.apply("Convert to JSON", convertToFailsafeAndMaybeApplyUdf(options));
PCollectionTuple splunkEventTuple = transformedLines.get(UDF_OUT).apply("Convert to Splunk Event", convertToSplunkEvent());
// Assert
PAssert.that(transformedLines.get(UDF_OUT)).satisfies(collection -> {
FailsafeElement element = collection.iterator().next();
assertThat(element.getPayload()).isEqualTo(stringifiedJsonRecord);
return null;
});
PAssert.that(transformedLines.get(UDF_ERROR_OUT)).empty();
PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_OUT)).containsInAnyOrder(expectedSplunkEvent);
PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_ERROR_OUT)).empty();
// Execute pipeline
pipeline.run();
}
use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.
the class GCSToSplunkTest method testGCSToSplunkReadUdf.
@Test
public void testGCSToSplunkReadUdf() {
// Arrange
String stringifiedJsonRecord = "{\"id\":\"007\",\"state\":\"CA\",\"price\":26.23}";
SplunkEvent expectedSplunkEvent = SplunkEvent.newBuilder().withEvent(stringifiedJsonRecord).create();
CoderRegistry coderRegistry = pipeline.getCoderRegistry();
coderRegistry.registerCoderForClass(SplunkEvent.class, SplunkEventCoder.of());
coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
options.setJavascriptTextTransformGcsPath(TRANSFORM_FILE_PATH);
options.setJavascriptTextTransformFunctionName("transform");
options.setContainsHeaders(false);
options.setInputFileSpec(NO_HEADER_CSV_FILE_PATH);
// Act
PCollectionTuple readCsvOut = pipeline.apply("Read CSV", readFromCsv(options));
PCollectionTuple transformedLines = readCsvOut.apply("Convert to JSON", convertToFailsafeAndMaybeApplyUdf(options));
PCollectionTuple splunkEventTuple = transformedLines.get(UDF_OUT).apply("Convert to Splunk Event", convertToSplunkEvent());
// Assert
PAssert.that(transformedLines.get(UDF_OUT)).satisfies(collection -> {
FailsafeElement element = collection.iterator().next();
assertThat(element.getPayload()).isEqualTo(stringifiedJsonRecord);
return null;
});
PAssert.that(transformedLines.get(UDF_ERROR_OUT)).empty();
PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_OUT)).containsInAnyOrder(expectedSplunkEvent);
PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_ERROR_OUT)).empty();
// Execute pipeline
pipeline.run();
}
use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.
the class GCSToSplunkTest method testGCSToSplunkReadJsonSchema.
@Test
public void testGCSToSplunkReadJsonSchema() {
// Arrange
String stringifiedJsonRecord = "{\"id\":\"007\",\"state\":\"CA\",\"price\":26.23}";
SplunkEvent expectedSplunkEvent = SplunkEvent.newBuilder().withEvent(stringifiedJsonRecord).create();
CoderRegistry coderRegistry = pipeline.getCoderRegistry();
coderRegistry.registerCoderForClass(SplunkEvent.class, SplunkEventCoder.of());
coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
options.setJsonSchemaPath(JSON_SCHEMA_FILE_PATH);
options.setContainsHeaders(false);
options.setInputFileSpec(NO_HEADER_CSV_FILE_PATH);
// Act
PCollectionTuple readCsvOut = pipeline.apply("Read CSV", readFromCsv(options));
PCollectionTuple transformedLines = readCsvOut.apply("Convert to JSON", convertToFailsafeAndMaybeApplyUdf(options));
PCollectionTuple splunkEventTuple = transformedLines.get(UDF_OUT).apply("Convert to Splunk Event", convertToSplunkEvent());
// Assert
PAssert.that(transformedLines.get(UDF_OUT)).satisfies(collection -> {
FailsafeElement element = collection.iterator().next();
assertThat(element.getPayload()).isEqualTo(stringifiedJsonRecord);
return null;
});
PAssert.that(transformedLines.get(UDF_ERROR_OUT)).empty();
PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_OUT)).containsInAnyOrder(expectedSplunkEvent);
PAssert.that(splunkEventTuple.get(SPLUNK_EVENT_ERROR_OUT)).empty();
// Execute pipeline
pipeline.run();
}
use of com.google.cloud.teleport.v2.templates.GCSToSplunk.GCSToSplunkOptions in project DataflowTemplates by GoogleCloudPlatform.
the class GCSToSplunkTest method testGCSToSplunkWriteErrorsToFolder.
@Test
public void testGCSToSplunkWriteErrorsToFolder() throws IOException {
// Arrange
String stringifiedSplunkError = "Payload: test-payload. Error Message: test-message. Splunk write status code: 123.";
PCollection<String> stringifiedErrorCollection = pipeline.apply("Add Stringified Errors", Create.of(stringifiedSplunkError).withCoder(StringUtf8Coder.of()));
PCollectionTuple stringifiedErrorTuple = PCollectionTuple.of(COMBINED_ERRORS, stringifiedErrorCollection);
GCSToSplunkOptions options = PipelineOptionsFactory.create().as(GCSToSplunkOptions.class);
options.setInvalidOutputPath(tmpFolder.getRoot().getAbsolutePath() + "errors.txt");
// Act
stringifiedErrorTuple.apply("Output Errors To GCS", writeErrorsToGCS(options));
// Execute pipeline
pipeline.run();
// Assert
File file = new File(tmpFolder.getRoot().getAbsolutePath() + "errors.txt-00000-of-00001");
String fileContents = Files.toString(file, Charsets.UTF_8);
assertThat(fileContents).contains(stringifiedSplunkError);
}
Aggregations