Examples with InvalidChangeEventException com.google.cloud.teleport.v2.templates.datastream.InvalidChangeEventException used on opensource projects

Example 1 with InvalidChangeEventException

use of com.google.cloud.teleport.v2.templates.datastream.InvalidChangeEventException in project DataflowTemplates by GoogleCloudPlatform.

the class SpannerTransactionWriterDoFn method processElement.

@ProcessElement
public void processElement(ProcessContext c) {
    FailsafeElement<String, String> msg = c.element();
    Ddl ddl = c.sideInput(ddlView);
    processedEvents.inc();
    /*
     * Try Catch block to capture any exceptions that might occur while processing
     * DataStream events while writing to Cloud Spanner. All Exceptions that are caught
     * can be retried based on the exception type.
     */
    try {
        JsonNode changeEvent = mapper.readTree(msg.getPayload());
        ChangeEventContext changeEventContext = ChangeEventContextFactory.createChangeEventContext(changeEvent, ddl, shadowTablePrefix, sourceType);
        // Sequence information for the current change event.
        ChangeEventSequence currentChangeEventSequence = ChangeEventSequenceFactory.createChangeEventSequenceFromChangeEventContext(changeEventContext);
        // Start transaction
        spannerAccessor.getDatabaseClient().readWriteTransaction().run((TransactionCallable<Void>) transaction -> {
            ChangeEventSequence previousChangeEventSequence = ChangeEventSequenceFactory.createChangeEventSequenceFromShadowTable(transaction, changeEventContext);
            if (previousChangeEventSequence != null && previousChangeEventSequence.compareTo(currentChangeEventSequence) >= 0) {
                return null;
            }
            transaction.buffer(changeEventContext.getMutations());
            return null;
        });
        com.google.cloud.Timestamp timestamp = com.google.cloud.Timestamp.now();
        c.output(timestamp);
        sucessfulEvents.inc();
    } catch (InvalidChangeEventException e) {
        // Errors that result from invalid change events.
        outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
        skippedEvents.inc();
    } catch (ChangeEventConvertorException e) {
        // Errors that result during Event conversions are not retryable.
        outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
        conversionErrors.inc();
    } catch (SpannerException se) {
        /* Errors that happen when writing to Cloud Spanner are considered retryable.
       * Since all event convertion errors are caught beforehand as permanent errors,
       * any other errors encountered while writing to Cloud Spanner can be retried.
       * Examples include:
       * 1. Deadline exceeded errors from Cloud Spanner.
       * 2. Failures due to foreign key/interleaved table constraints.
       * 3. Any transient errors in Cloud Spanner.
       */
        outputWithErrorTag(c, msg, se, SpannerTransactionWriter.RETRYABLE_ERROR_TAG);
        retryableErrors.inc();
    } catch (Exception e) {
        // Any other errors are considered severe and not retryable.
        outputWithErrorTag(c, msg, e, SpannerTransactionWriter.PERMANENT_ERROR_TAG);
        failedEvents.inc();
    }
}

Example 2 with InvalidChangeEventException

use of com.google.cloud.teleport.v2.templates.datastream.InvalidChangeEventException in project DataflowTemplates by GoogleCloudPlatform.

the class ChangeEventConvertor method changeEventToInsertOrUpdateMutation.

private static Mutation changeEventToInsertOrUpdateMutation(Ddl ddl, JsonNode changeEvent) throws ChangeEventConvertorException, InvalidChangeEventException {
    String tableName = changeEvent.get(DatastreamConstants.EVENT_TABLE_NAME_KEY).asText();
    List<String> changeEventKeys = getEventColumnKeys(changeEvent);
    try {
        Table table = ddl.table(tableName);
        Mutation.WriteBuilder builder = Mutation.newInsertOrUpdateBuilder(table.name());
        Set<String> keyColumns = table.primaryKeys().stream().map(keyCol -> keyCol.name()).map(colName -> colName.toLowerCase()).collect(Collectors.toSet());
        populateMutationBuilderWithEvent(table, builder, changeEvent, changeEventKeys, keyColumns);
        return builder.build();
    } catch (Exception e) {
        throw new ChangeEventConvertorException(e);
    }
}