use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventContext in project DataflowTemplates by GoogleCloudPlatform.
the class SpannerTransactionWriterDoFn method processElement.
@ProcessElement
public void processElement(ProcessContext c) {
FailsafeElement<String, String> msg = c.element();
Ddl ddl = c.sideInput(ddlView);
processedEvents.inc();
/*
* Try Catch block to capture any exceptions that might occur while processing
* DataStream events while writing to Cloud Spanner. All Exceptions that are caught
* can be retried based on the exception type.
*/
try {
JsonNode changeEvent = mapper.readTree(msg.getPayload());
ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(changeEvent, ddl, shadowTablePrefix, sourceType);
// Sequence information for the current change event.
ChangeEventSequence currentChangeEventSequence = ChangeEventSequenceFactory.createChangeEventSequenceFromChangeEventContext(changeEventContext);
// Start transaction
spannerAccessor.getDatabaseClient().readWriteTransaction().run((TransactionCallable<Void>) transaction -> {
ChangeEventSequence previousChangeEventSequence = ChangeEventSequenceFactory.createChangeEventSequenceFromShadowTable(transaction, changeEventContext);
if (previousChangeEventSequence != null && previousChangeEventSequence.compareTo(currentChangeEventSequence) >= 0) {
return null;
}
transaction.buffer(changeEventContext.getMutations());
return null;
});
com.google.cloud.Timestamp timestamp = com.google.cloud.Timestamp.now();
c.output(timestamp);
sucessfulEvents.inc();
} catch (InvalidChangeEventException e) {
// Errors that result from invalid change events.
outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
skippedEvents.inc();
} catch (ChangeEventConvertorException e) {
// Errors that result during Event conversions are not retryable.
outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
conversionErrors.inc();
} catch (SpannerException se) {
/* Errors that happen when writing to Cloud Spanner are considered retryable.
* Since all event convertion errors are caught beforehand as permanent errors,
* any other errors encountered while writing to Cloud Spanner can be retried.
* Examples include:
* 1. Deadline exceeded errors from Cloud Spanner.
* 2. Failures due to foreign key/interleaved table constraints.
* 3. Any transient errors in Cloud Spanner.
*/
outputWithErrorTag(c, msg, se, SpannerTransactionWriter.RETRYABLE_ERROR_TAG);
retryableErrors.inc();
} catch (Exception e) {
// Any other errors are considered severe and not retryable.
outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
failedEvents.inc();
}
}
use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventContext in project DataflowTemplates by GoogleCloudPlatform.
the class OracleChangeEventContextTest method canGenerateShadowTableMutation.
@Test
public void canGenerateShadowTableMutation() throws Exception {
// Test Ddl
Ddl ddl = ChangeEventConvertorTest.getTestDdl();
// Test Change Event
JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
changeEvent.put(DatastreamConstants.ORACLE_TIMESTAMP_KEY, eventTimestamp);
changeEvent.put(DatastreamConstants.ORACLE_SCN_KEY, "1");
changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.ORACLE_SOURCE_TYPE);
ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.ORACLE_SOURCE_TYPE);
Mutation shadowMutation = changeEventContext.getShadowTableMutation();
Map<String, Value> actual = shadowMutation.asMap();
// Expected result
Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
expected.put(DatastreamConstants.ORACLE_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
expected.put(DatastreamConstants.ORACLE_SCN_SHADOW_INFO.getLeft(), Value.int64(1));
// Verify if OracleChangeEventContext was actually created.
assertThat(changeEventContext, instanceOf(OracleChangeEventContext.class));
// Verify shadow mutation
assertThat(actual, is(expected));
assertEquals(shadowMutation.getTable(), "shadow_Users2");
assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}
use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventContext in project DataflowTemplates by GoogleCloudPlatform.
the class MySqlChangeEventContextTest method canGenerateShadowTableMutation.
@Test
public void canGenerateShadowTableMutation() throws Exception {
long eventTimestamp = 1615159728L;
// Test Ddl
Ddl ddl = ChangeEventConvertorTest.getTestDdl();
// Test Change Event
JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
changeEvent.put(DatastreamConstants.MYSQL_TIMESTAMP_KEY, eventTimestamp);
changeEvent.put(DatastreamConstants.MYSQL_LOGFILE_KEY, "file1.log");
changeEvent.put(DatastreamConstants.MYSQL_LOGPOSITION_KEY, 1L);
changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.MYSQL_SOURCE_TYPE);
ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.MYSQL_SOURCE_TYPE);
Mutation shadowMutation = changeEventContext.getShadowTableMutation();
Map<String, Value> actual = shadowMutation.asMap();
// Expected result
Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
expected.put(DatastreamConstants.MYSQL_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
expected.put(DatastreamConstants.MYSQL_LOGFILE_SHADOW_INFO.getLeft(), Value.string("file1.log"));
expected.put(DatastreamConstants.MYSQL_LOGPOSITION_SHADOW_INFO.getLeft(), Value.int64(1));
// Verify if MySqlChangeEventContext was actually created.
assertThat(changeEventContext, instanceOf(MySqlChangeEventContext.class));
// Verify shadow mutation
assertThat(actual, is(expected));
assertEquals(shadowMutation.getTable(), "shadow_Users2");
assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}
use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventContext in project DataflowTemplates by GoogleCloudPlatform.
the class MySqlChangeEventContextTest method canGenerateShadowTableMutationForBackfillEventsWithMissingSortOrderKeys.
@Test
public void canGenerateShadowTableMutationForBackfillEventsWithMissingSortOrderKeys() throws Exception {
long eventTimestamp = 1615159728L;
// Test Ddl
Ddl ddl = ChangeEventConvertorTest.getTestDdl();
// Test Change Event which does not contain sort order fields like log file and log position.
JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
changeEvent.put(DatastreamConstants.MYSQL_TIMESTAMP_KEY, eventTimestamp);
changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.MYSQL_SOURCE_TYPE);
ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.MYSQL_SOURCE_TYPE);
Mutation shadowMutation = changeEventContext.getShadowTableMutation();
Map<String, Value> actual = shadowMutation.asMap();
// Expected result
Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
expected.put(DatastreamConstants.MYSQL_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
expected.put(DatastreamConstants.MYSQL_LOGFILE_SHADOW_INFO.getLeft(), Value.string(""));
expected.put(DatastreamConstants.MYSQL_LOGPOSITION_SHADOW_INFO.getLeft(), Value.int64(-1));
// Verify if MySqlChangeEventContext was actually created.
assertThat(changeEventContext, instanceOf(MySqlChangeEventContext.class));
// Verify shadow mutation
assertThat(actual, is(expected));
assertEquals(shadowMutation.getTable(), "shadow_Users2");
assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}
use of com.google.cloud.teleport.v2.templates.datastream.ChangeEventContext in project DataflowTemplates by GoogleCloudPlatform.
the class MySqlChangeEventContextTest method canGenerateShadowTableMutationForBackfillEvents.
@Test
public void canGenerateShadowTableMutationForBackfillEvents() throws Exception {
long eventTimestamp = 1615159728L;
// Test Ddl
Ddl ddl = ChangeEventConvertorTest.getTestDdl();
// Test Change Event
JSONObject changeEvent = ChangeEventConvertorTest.getTestChangeEvent("Users2");
changeEvent.put(DatastreamConstants.MYSQL_TIMESTAMP_KEY, eventTimestamp);
changeEvent.put(DatastreamConstants.MYSQL_LOGFILE_KEY, JSONObject.NULL);
changeEvent.put(DatastreamConstants.MYSQL_LOGPOSITION_KEY, JSONObject.NULL);
changeEvent.put(DatastreamConstants.EVENT_SOURCE_TYPE_KEY, DatastreamConstants.MYSQL_SOURCE_TYPE);
ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(getJsonNode(changeEvent.toString()), ddl, "shadow_", DatastreamConstants.MYSQL_SOURCE_TYPE);
Mutation shadowMutation = changeEventContext.getShadowTableMutation();
Map<String, Value> actual = shadowMutation.asMap();
// Expected result
Map<String, Value> expected = ChangeEventConvertorTest.getExpectedMapForTestChangeEvent();
expected.put(DatastreamConstants.MYSQL_TIMESTAMP_SHADOW_INFO.getLeft(), Value.int64(eventTimestamp));
expected.put(DatastreamConstants.MYSQL_LOGFILE_SHADOW_INFO.getLeft(), Value.string(""));
expected.put(DatastreamConstants.MYSQL_LOGPOSITION_SHADOW_INFO.getLeft(), Value.int64(-1));
// Verify if MySqlChangeEventContext was actually created.
assertThat(changeEventContext, instanceOf(MySqlChangeEventContext.class));
// Verify shadow mutation
assertThat(actual, is(expected));
assertEquals(shadowMutation.getTable(), "shadow_Users2");
assertEquals(shadowMutation.getOperation(), Mutation.Op.INSERT_OR_UPDATE);
}
Aggregations