use of com.google.cloud.teleport.v2.templates.spanner.ddl.IndexColumn in project DataflowTemplates by GoogleCloudPlatform.
the class ChangeEventConvertor method changeEventToPrimaryKey.
static com.google.cloud.spanner.Key changeEventToPrimaryKey(Ddl ddl, JsonNode changeEvent) throws ChangeEventConvertorException {
String tableName = changeEvent.get(DatastreamConstants.EVENT_TABLE_NAME_KEY).asText();
try {
Table table = ddl.table(tableName);
ImmutableList<IndexColumn> keyColumns = table.primaryKeys();
com.google.cloud.spanner.Key.Builder pk = com.google.cloud.spanner.Key.newBuilder();
for (IndexColumn keyColumn : keyColumns) {
Column key = table.column(keyColumn.name());
Type keyColType = key.type();
String keyColName = key.name().toLowerCase();
switch(keyColType.getCode()) {
case BOOL:
pk.append(ChangeEventTypeConvertor.toBoolean(changeEvent, keyColName, /*requiredField=*/
true));
break;
case INT64:
pk.append(ChangeEventTypeConvertor.toLong(changeEvent, keyColName, /*requiredField=*/
true));
break;
case FLOAT64:
pk.append(ChangeEventTypeConvertor.toDouble(changeEvent, keyColName, /*requiredField=*/
true));
break;
case STRING:
pk.append(ChangeEventTypeConvertor.toString(changeEvent, keyColName, /*requiredField=*/
true));
break;
case NUMERIC:
pk.append(ChangeEventTypeConvertor.toNumericBigDecimal(changeEvent, keyColName, /*requiredField=*/
true));
break;
case BYTES:
pk.append(ChangeEventTypeConvertor.toByteArray(changeEvent, keyColName, /*requiredField=*/
true));
break;
case TIMESTAMP:
pk.append(ChangeEventTypeConvertor.toTimestamp(changeEvent, keyColName, /*requiredField=*/
true));
break;
case DATE:
pk.append(ChangeEventTypeConvertor.toDate(changeEvent, keyColName, /*requiredField=*/
true));
break;
// TODO(b/179070999) - Add support for other data types.
default:
throw new IllegalArgumentException("Column name(" + keyColName + ") has unsupported column type(" + keyColType + ")");
}
}
return pk.build();
} catch (Exception e) {
throw new ChangeEventConvertorException(e);
}
}
use of com.google.cloud.teleport.v2.templates.spanner.ddl.IndexColumn in project DataflowTemplates by GoogleCloudPlatform.
the class ChangeEventConvertor method changeEventToShadowTableMutationBuilder.
static Mutation.WriteBuilder changeEventToShadowTableMutationBuilder(Ddl ddl, JsonNode changeEvent, String shadowTablePrefix) throws ChangeEventConvertorException {
String tableName = changeEvent.get(DatastreamConstants.EVENT_TABLE_NAME_KEY).asText();
String shadowTableName = shadowTablePrefix + tableName;
try {
Table table = ddl.table(shadowTableName);
ImmutableList<IndexColumn> keyColumns = table.primaryKeys();
List<String> keyColumnNames = keyColumns.stream().map(IndexColumn::name).map(colName -> colName.toLowerCase()).collect(Collectors.toList());
Set<String> requiredKeyColumnNames = new HashSet<>(keyColumnNames);
Mutation.WriteBuilder builder = Mutation.newInsertOrUpdateBuilder(shadowTableName);
populateMutationBuilderWithEvent(table, builder, changeEvent, keyColumnNames, requiredKeyColumnNames);
return builder;
} catch (Exception e) {
throw new ChangeEventConvertorException(e);
}
}
use of com.google.cloud.teleport.v2.templates.spanner.ddl.IndexColumn in project DataflowTemplates by GoogleCloudPlatform.
the class ShadowTableCreator method constructShadowTable.
/*
* Constructs a shadow table for a data table in the information schema.
* Note: Shadow tables for interleaved tables are not interleaved to
* their shadow parent table.
*/
Table constructShadowTable(Ddl informationSchema, String dataTableName) {
// Create a new shadow table with the given prefix.
Table.Builder shadowTableBuilder = Table.builder();
String shadowTableName = shadowTablePrefix + dataTableName;
shadowTableBuilder.name(shadowTableName);
// Add key columns from the data table to the shadow table builder.
Table dataTable = informationSchema.table(dataTableName);
Set<String> primaryKeyColNames = dataTable.primaryKeys().stream().map(k -> k.name()).collect(Collectors.toSet());
List<Column> primaryKeyCols = dataTable.columns().stream().filter(col -> primaryKeyColNames.contains(col.name())).collect(Collectors.toList());
for (Column col : primaryKeyCols) {
shadowTableBuilder.addColumn(col);
}
// Add primary key constraints.
for (IndexColumn keyColumn : dataTable.primaryKeys()) {
if (keyColumn.order() == IndexColumn.Order.ASC) {
shadowTableBuilder.primaryKey().asc(keyColumn.name()).end();
} else if (keyColumn.order() == IndexColumn.Order.DESC) {
shadowTableBuilder.primaryKey().desc(keyColumn.name()).end();
}
}
// Add extra column to track ChangeEventSequence information
addChangeEventSequenceColumns(shadowTableBuilder);
return shadowTableBuilder.build();
}
Aggregations