use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventConvertorException in project DataflowTemplates by GoogleCloudPlatform.
the class SpannerTransactionWriterDoFn method processElement.
@ProcessElement
public void processElement(ProcessContext c) {
FailsafeElement<String, String> msg = c.element();
Ddl ddl = c.sideInput(ddlView);
processedEvents.inc();
/*
* Try Catch block to capture any exceptions that might occur while processing
* DataStream events while writing to Cloud Spanner. All Exceptions that are caught
* can be retried based on the exception type.
*/
try {
JsonNode changeEvent = mapper.readTree(msg.getPayload());
ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(changeEvent, ddl, shadowTablePrefix, sourceType);
// Sequence information for the current change event.
ChangeEventSequence currentChangeEventSequence = ChangeEventSequenceFactory.createChangeEventSequenceFromChangeEventContext(changeEventContext);
// Start transaction
spannerAccessor.getDatabaseClient().readWriteTransaction().run((TransactionCallable<Void>) transaction -> {
ChangeEventSequence previousChangeEventSequence = ChangeEventSequenceFactory.createChangeEventSequenceFromShadowTable(transaction, changeEventContext);
if (previousChangeEventSequence != null && previousChangeEventSequence.compareTo(currentChangeEventSequence) >= 0) {
return null;
}
transaction.buffer(changeEventContext.getMutations());
return null;
});
com.google.cloud.Timestamp timestamp = com.google.cloud.Timestamp.now();
c.output(timestamp);
sucessfulEvents.inc();
} catch (InvalidChangeEventException e) {
// Errors that result from invalid change events.
outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
skippedEvents.inc();
} catch (ChangeEventConvertorException e) {
// Errors that result during Event conversions are not retryable.
outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
conversionErrors.inc();
} catch (SpannerException se) {
/* Errors that happen when writing to Cloud Spanner are considered retryable.
* Since all event convertion errors are caught beforehand as permanent errors,
* any other errors encountered while writing to Cloud Spanner can be retried.
* Examples include:
* 1. Deadline exceeded errors from Cloud Spanner.
* 2. Failures due to foreign key/interleaved table constraints.
* 3. Any transient errors in Cloud Spanner.
*/
outputWithErrorTag(c, msg, se, SpannerTransactionWriter.RETRYABLE_ERROR_TAG);
retryableErrors.inc();
} catch (Exception e) {
// Any other errors are considered severe and not retryable.
outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
failedEvents.inc();
}
}
use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventConvertorException in project DataflowTemplates by GoogleCloudPlatform.
the class ChangeEventConvertor method changeEventToPrimaryKey.
static com.google.cloud.spanner.Key changeEventToPrimaryKey(Ddl ddl, JsonNode changeEvent) throws ChangeEventConvertorException {
String tableName = changeEvent.get(DatastreamConstants.EVENT_TABLE_NAME_KEY).asText();
try {
Table table = ddl.table(tableName);
ImmutableList<IndexColumn> keyColumns = table.primaryKeys();
com.google.cloud.spanner.Key.Builder pk = com.google.cloud.spanner.Key.newBuilder();
for (IndexColumn keyColumn : keyColumns) {
Column key = table.column(keyColumn.name());
Type keyColType = key.type();
String keyColName = key.name().toLowerCase();
switch(keyColType.getCode()) {
case BOOL:
pk.append(ChangeEventTypeConvertor.toBoolean(changeEvent, keyColName, /*requiredField=*/
true));
break;
case INT64:
pk.append(ChangeEventTypeConvertor.toLong(changeEvent, keyColName, /*requiredField=*/
true));
break;
case FLOAT64:
pk.append(ChangeEventTypeConvertor.toDouble(changeEvent, keyColName, /*requiredField=*/
true));
break;
case STRING:
pk.append(ChangeEventTypeConvertor.toString(changeEvent, keyColName, /*requiredField=*/
true));
break;
case NUMERIC:
pk.append(ChangeEventTypeConvertor.toNumericBigDecimal(changeEvent, keyColName, /*requiredField=*/
true));
break;
case BYTES:
pk.append(ChangeEventTypeConvertor.toByteArray(changeEvent, keyColName, /*requiredField=*/
true));
break;
case TIMESTAMP:
pk.append(ChangeEventTypeConvertor.toTimestamp(changeEvent, keyColName, /*requiredField=*/
true));
break;
case DATE:
pk.append(ChangeEventTypeConvertor.toDate(changeEvent, keyColName, /*requiredField=*/
true));
break;
// TODO(b/179070999) - Add support for other data types.
default:
throw new IllegalArgumentException("Column name(" + keyColName + ") has unsupported column type(" + keyColType + ")");
}
}
return pk.build();
} catch (Exception e) {
throw new ChangeEventConvertorException(e);
}
}
use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventConvertorException in project DataflowTemplates by GoogleCloudPlatform.
the class ChangeEventConvertor method changeEventToInsertOrUpdateMutation.
private static Mutation changeEventToInsertOrUpdateMutation(Ddl ddl, JsonNode changeEvent) throws ChangeEventConvertorException, InvalidChangeEventException {
String tableName = changeEvent.get(DatastreamConstants.EVENT_TABLE_NAME_KEY).asText();
List<String> changeEventKeys = getEventColumnKeys(changeEvent);
try {
Table table = ddl.table(tableName);
Mutation.WriteBuilder builder = Mutation.newInsertOrUpdateBuilder(table.name());
Set<String> keyColumns = table.primaryKeys().stream().map(keyCol -> keyCol.name()).map(colName -> colName.toLowerCase()).collect(Collectors.toSet());
populateMutationBuilderWithEvent(table, builder, changeEvent, changeEventKeys, keyColumns);
return builder.build();
} catch (Exception e) {
throw new ChangeEventConvertorException(e);
}
}
use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventConvertorException in project DataflowTemplates by GoogleCloudPlatform.
the class ChangeEventConvertor method changeEventToShadowTableMutationBuilder.
static Mutation.WriteBuilder changeEventToShadowTableMutationBuilder(Ddl ddl, JsonNode changeEvent, String shadowTablePrefix) throws ChangeEventConvertorException {
String tableName = changeEvent.get(DatastreamConstants.EVENT_TABLE_NAME_KEY).asText();
String shadowTableName = shadowTablePrefix + tableName;
try {
Table table = ddl.table(shadowTableName);
ImmutableList<IndexColumn> keyColumns = table.primaryKeys();
List<String> keyColumnNames = keyColumns.stream().map(IndexColumn::name).map(colName -> colName.toLowerCase()).collect(Collectors.toList());
Set<String> requiredKeyColumnNames = new HashSet<>(keyColumnNames);
Mutation.WriteBuilder builder = Mutation.newInsertOrUpdateBuilder(shadowTableName);
populateMutationBuilderWithEvent(table, builder, changeEvent, keyColumnNames, requiredKeyColumnNames);
return builder;
} catch (Exception e) {
throw new ChangeEventConvertorException(e);
}
}
Aggregations