Search in sources :

Example 51 with Table

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class SpannerUtils method getSpannerTableByName.

/**
 * @return a map where the key is the table name tracked by the change stream and the value is the
 *     {@link SpannerTable} object of the table name.
 */
private Map<String, TrackedSpannerTable> getSpannerTableByName(Set<String> spannerTableNames, Map<String, Set<String>> spannerColumnNamesExplicitlyTrackedByChangeStreamByTableName) {
    Map<String, Set<String>> keyColumnNameByTableName = getKeyColumnNameByTableName(spannerTableNames);
    Map<String, List<TrackedSpannerColumn>> spannerColumnsByTableName = getSpannerColumnsByTableName(spannerTableNames, keyColumnNameByTableName, spannerColumnNamesExplicitlyTrackedByChangeStreamByTableName);
    Map<String, TrackedSpannerTable> result = new HashMap<>();
    for (String tableName : spannerColumnsByTableName.keySet()) {
        List<TrackedSpannerColumn> pkColumns = new ArrayList<>();
        List<TrackedSpannerColumn> nonPkColumns = new ArrayList<>();
        Set<String> keyColumnNames = keyColumnNameByTableName.get(tableName);
        for (TrackedSpannerColumn spannerColumn : spannerColumnsByTableName.get(tableName)) {
            if (keyColumnNames.contains(spannerColumn.getName())) {
                pkColumns.add(spannerColumn);
            } else {
                nonPkColumns.add(spannerColumn);
            }
        }
        result.put(tableName, new TrackedSpannerTable(tableName, pkColumns, nonPkColumns));
    }
    return result;
}
Also used : TrackedSpannerTable(com.google.cloud.teleport.v2.templates.spannerchangestreamstobigquery.model.TrackedSpannerTable) TrackedSpannerColumn(com.google.cloud.teleport.v2.templates.spannerchangestreamstobigquery.model.TrackedSpannerColumn) Set(java.util.Set) HashSet(java.util.HashSet) ResultSet(com.google.cloud.spanner.ResultSet) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List)

Aggregations

Test (org.junit.Test)26 BigQueryTable (com.google.cloud.teleport.v2.values.BigQueryTable)15 BigQueryTablePartition (com.google.cloud.teleport.v2.values.BigQueryTablePartition)12 Filter (com.google.cloud.teleport.v2.utils.BigQueryMetadataLoader.Filter)10 ArrayList (java.util.ArrayList)10 FailsafeElement (com.google.cloud.teleport.v2.values.FailsafeElement)9 Pipeline (org.apache.beam.sdk.Pipeline)9 Ddl (com.google.cloud.teleport.v2.templates.spanner.ddl.Ddl)8 Table (com.google.cloud.teleport.v2.templates.spanner.ddl.Table)8 Set (java.util.Set)8 PipelineResult (org.apache.beam.sdk.PipelineResult)8 TableRow (com.google.api.services.bigquery.model.TableRow)6 IntegrationTest (com.google.cloud.teleport.v2.spanner.IntegrationTest)6 PCollection (org.apache.beam.sdk.values.PCollection)6 PCollectionTuple (org.apache.beam.sdk.values.PCollectionTuple)6 IOException (java.io.IOException)5 WriteResult (org.apache.beam.sdk.io.gcp.bigquery.WriteResult)5 Timestamp (com.google.cloud.Timestamp)4 Column (com.google.cloud.teleport.v2.templates.spanner.ddl.Column)4 IndexColumn (com.google.cloud.teleport.v2.templates.spanner.ddl.IndexColumn)4