Examples with SplunkWriteError org.apache.beam.sdk.io.splunk.SplunkWriteError used on opensource projects

Example 1 with SplunkWriteError

use of org.apache.beam.sdk.io.splunk.SplunkWriteError in project DataflowTemplates by GoogleCloudPlatform.

the class GCSToSplunk method run.

/**
 * Runs the pipeline to completion with the specified options.
 *
 * @param options The execution options.
 * @return The pipeline result.
 */
public static PipelineResult run(GCSToSplunkOptions options) {
    Pipeline pipeline = Pipeline.create(options);
    CoderRegistry registry = pipeline.getCoderRegistry();
    registry.registerCoderForClass(SplunkEvent.class, SplunkEventCoder.of());
    registry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
    PCollectionTuple readCsvTuple = pipeline.apply("Read CSV", readFromCsv(options));
    PCollectionTuple failsafeTransformedLines = readCsvTuple.apply("Convert To JSON", convertToFailsafeAndMaybeApplyUdf(options));
    PCollectionTuple splunkEventTuple = failsafeTransformedLines.get(UDF_OUT).apply("Convert to Splunk Event", convertToSplunkEvent());
    PCollection<SplunkWriteError> wrappedSplunkWriteErrors = splunkEventTuple.get(SPLUNK_EVENT_OUT).apply("Write to Splunk", writeToSplunk(options));
    flattenErrorsAndConvertToString(failsafeTransformedLines.get(UDF_ERROR_OUT), splunkEventTuple.get(SPLUNK_EVENT_ERROR_OUT), wrappedSplunkWriteErrors).apply("Output Errors To GCS", writeErrorsToGCS(options));
    return pipeline.run();
}

Example 2 with SplunkWriteError

use of org.apache.beam.sdk.io.splunk.SplunkWriteError in project DataflowTemplates by GoogleCloudPlatform.

the class SplunkWriteErrorCoderTest method testEncodeDecode.

@Test
public void testEncodeDecode() throws IOException {
    String payload = "test-payload";
    String message = "test-message";
    Integer statusCode = 123;
    SplunkWriteError actualError = SplunkWriteError.newBuilder().withPayload(payload).withStatusCode(statusCode).withStatusMessage(message).create();
    SplunkWriteErrorCoder coder = SplunkWriteErrorCoder.of();
    try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
        coder.encode(actualError, bos);
        try (ByteArrayInputStream bin = new ByteArrayInputStream(bos.toByteArray())) {
            SplunkWriteError decodedWriteError = coder.decode(bin);
            assertThat(decodedWriteError).isEqualTo(actualError);
        }
    }
}

Example 3 with SplunkWriteError

use of org.apache.beam.sdk.io.splunk.SplunkWriteError in project DataflowTemplates by GoogleCloudPlatform.

the class GCSToSplunkTest method testGCSToSplunkConvertWriteErrors.

@Test
public void testGCSToSplunkConvertWriteErrors() {
    // Arrange
    String stringifiedSplunkError = "Payload: test-payload. Error Message: test-message. Splunk write status code: 123.";
    String firstStringifiedFailsafeError = "Payload: world. Error Message: failed!.";
    String secondStringifiedFailsafeError = "Payload: one. Error Message: error!.";
    CoderRegistry coderRegistry = pipeline.getCoderRegistry();
    coderRegistry.registerCoderForType(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor(), FAILSAFE_ELEMENT_CODER);
    SplunkWriteError splunkWriteError = SplunkWriteError.newBuilder().withPayload("test-payload").withStatusCode(123).withStatusMessage("test-message").create();
    PCollection<SplunkWriteError> splunkErrorCollection = pipeline.apply("Add Splunk Errors", Create.of(splunkWriteError).withCoder(SplunkWriteErrorCoder.of()));
    FailsafeElement<String, String> firstFailsafeElement = FailsafeElement.of("hello", "world").setErrorMessage("failed!");
    PCollection<FailsafeElement<String, String>> firstFailsafeElementCollection = pipeline.apply("Add FailsafeElements to First", Create.of(firstFailsafeElement).withCoder(FAILSAFE_ELEMENT_CODER));
    FailsafeElement<String, String> secondFailsafeElement = FailsafeElement.of("another", "one").setErrorMessage("error!");
    PCollection<FailsafeElement<String, String>> secondFailsafeElementCollection = pipeline.apply("Add FailsafeElements to Second", Create.of(secondFailsafeElement).withCoder(FAILSAFE_ELEMENT_CODER));
    // Act
    PCollectionTuple stringifiedErrors = flattenErrorsAndConvertToString(firstFailsafeElementCollection, secondFailsafeElementCollection, splunkErrorCollection);
    // Assert
    PAssert.that(stringifiedErrors.get(COMBINED_ERRORS)).containsInAnyOrder(stringifiedSplunkError, firstStringifiedFailsafeError, secondStringifiedFailsafeError);
    // Execute pipeline
    pipeline.run();
}