Examples with BigQueryConverters com.google.cloud.teleport.v2.transforms.BigQueryConverters used on opensource projects

Example 1 with BigQueryConverters

use of com.google.cloud.teleport.v2.transforms.BigQueryConverters in project DataflowTemplates by GoogleCloudPlatform.

the class StreamingDataGeneratorWriteToBigQuery method expand.

@Override
public PDone expand(PCollection<byte[]> fakeMessages) {
    WriteResult writeResults = fakeMessages.apply("Write Json messsages", BigQueryIO.<byte[]>write().to(getPipelineOptions().getOutputTableSpec()).withMethod(Method.STREAMING_INSERTS).ignoreInsertIds().withCreateDisposition(CreateDisposition.CREATE_NEVER).withWriteDisposition(WriteDisposition.valueOf(getPipelineOptions().getWriteDisposition())).withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()).withExtendedErrorInfo().withFormatFunction((message) -> {
        TableRow row = null;
        try {
            row = TableRowJsonCoder.of().decode(new ByteArrayInputStream(message), Coder.Context.OUTER);
        } catch (IOException e) {
            throw new RuntimeException("Failed converting to TableRow with an error:" + e.getMessage());
        }
        return row;
    }));
    // Write errors to Dead Letter table
    writeResults.getFailedInsertsWithErr().apply("Convert to FailSafe Element", MapElements.into(FAILSAFE_ELEMENT_CODER.getEncodedTypeDescriptor()).via(BigQueryConverters::wrapBigQueryInsertError)).setCoder(FAILSAFE_ELEMENT_CODER).apply("Write Failed Records", ErrorConverters.WriteStringMessageErrors.newBuilder().setErrorRecordsTable(MoreObjects.firstNonNull(getPipelineOptions().getOutputDeadletterTable(), getPipelineOptions().getOutputTableSpec() + DEFAULT_DEADLETTER_TABLE_SUFFIX)).setErrorRecordsTableSchema(// i.e schema in above method
    SchemaUtils.DEADLETTER_SCHEMA).build());
    return PDone.in(fakeMessages.getPipeline());
}