Examples with Table com.google.cloud.teleport.spanner.ddl.Table used on opensource projects

Example 6 with Table

use of com.google.cloud.teleport.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class DdlToAvroSchemaConverter method convert.

public Collection<Schema> convert(Ddl ddl) {
    Collection<Schema> schemas = new ArrayList<>();
    for (Table table : ddl.allTables()) {
        SchemaBuilder.RecordBuilder<Schema> recordBuilder = SchemaBuilder.record(table.name()).namespace(this.namespace);
        recordBuilder.prop("googleFormatVersion", version);
        recordBuilder.prop("googleStorage", "CloudSpanner");
        if (table.interleaveInParent() != null) {
            recordBuilder.prop("spannerParent", table.interleaveInParent());
            recordBuilder.prop("spannerOnDeleteAction", table.onDeleteCascade() ? "cascade" : "no action");
        }
        if (table.dialect() == Dialect.GOOGLE_STANDARD_SQL) {
            if (table.primaryKeys() != null) {
                String encodedPk = table.primaryKeys().stream().map(IndexColumn::prettyPrint).collect(Collectors.joining(","));
                recordBuilder.prop("spannerPrimaryKey", encodedPk);
            }
            for (int i = 0; i < table.primaryKeys().size(); i++) {
                recordBuilder.prop("spannerPrimaryKey_" + i, table.primaryKeys().get(i).prettyPrint());
            }
        } else if (table.dialect() == Dialect.POSTGRESQL) {
            if (table.primaryKeys() != null) {
                String encodedPk = table.primaryKeys().stream().map(c -> "\"" + c.name() + "\"").collect(Collectors.joining(", "));
                recordBuilder.prop("spannerPrimaryKey", encodedPk);
            }
            for (int i = 0; i < table.primaryKeys().size(); i++) {
                IndexColumn pk = table.primaryKeys().get(i);
                recordBuilder.prop("spannerPrimaryKey_" + i, "\"" + pk.name() + "\" ASC");
            }
        }
        for (int i = 0; i < table.indexes().size(); i++) {
            recordBuilder.prop("spannerIndex_" + i, table.indexes().get(i));
        }
        for (int i = 0; i < table.foreignKeys().size(); i++) {
            recordBuilder.prop("spannerForeignKey_" + i, table.foreignKeys().get(i));
        }
        for (int i = 0; i < table.checkConstraints().size(); i++) {
            recordBuilder.prop("spannerCheckConstraint_" + i, table.checkConstraints().get(i));
        }
        SchemaBuilder.FieldAssembler<Schema> fieldsAssembler = recordBuilder.fields();
        for (Column cm : table.columns()) {
            SchemaBuilder.FieldBuilder<Schema> fieldBuilder = fieldsAssembler.name(cm.name());
            fieldBuilder.prop("sqlType", cm.typeString());
            for (int i = 0; i < cm.columnOptions().size(); i++) {
                fieldBuilder.prop("spannerOption_" + i, cm.columnOptions().get(i));
            }
            if (cm.isGenerated()) {
                fieldBuilder.prop("notNull", Boolean.toString(cm.notNull()));
                fieldBuilder.prop("generationExpression", cm.generationExpression());
                fieldBuilder.prop("stored", Boolean.toString(cm.isStored()));
                // Make the type null to allow us not export the generated column values,
                // which are semantically logical entities.
                fieldBuilder.type(SchemaBuilder.builder().nullType()).withDefault(null);
            } else {
                if (cm.defaultExpression() != null) {
                    fieldBuilder.prop("defaultExpression", cm.defaultExpression());
                }
                Schema avroType = avroType(cm.type());
                if (!cm.notNull()) {
                    avroType = wrapAsNullable(avroType);
                }
                fieldBuilder.type(avroType).noDefault();
            }
        }
        Schema schema = fieldsAssembler.endRecord();
        schemas.add(schema);
    }
    for (View view : ddl.views()) {
        SchemaBuilder.RecordBuilder<Schema> recordBuilder = SchemaBuilder.record(view.name()).namespace(this.namespace);
        recordBuilder.prop("googleFormatVersion", version);
        recordBuilder.prop("googleStorage", "CloudSpanner");
        recordBuilder.prop("spannerViewQuery", view.query());
        if (view.security() != null) {
            recordBuilder.prop("spannerViewSecurity", view.security().toString());
        }
        schemas.add(recordBuilder.fields().endRecord());
    }
    for (ChangeStream changeStream : ddl.changeStreams()) {
        SchemaBuilder.RecordBuilder<Schema> recordBuilder = SchemaBuilder.record(changeStream.name()).namespace(this.namespace);
        recordBuilder.prop("googleFormatVersion", version);
        recordBuilder.prop("googleStorage", "CloudSpanner");
        recordBuilder.prop(AvroUtil.CHANGE_STREAM_FOR_CLAUSE, changeStream.forClause() == null ? "" : changeStream.forClause());
        if (changeStream.options() != null) {
            for (int i = 0; i < changeStream.options().size(); i++) {
                recordBuilder.prop("spannerOption_" + i, changeStream.options().get(i));
            }
        }
        schemas.add(recordBuilder.fields().endRecord());
    }
    return schemas;
}

Example 7 with Table

use of com.google.cloud.teleport.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class ExportRelatedTablesCheckTest method randomExportTest.

/* Validates behavior of table filtering using randomly generated table and randomly selected
   * export tables */
@Test
public void randomExportTest() throws Exception {
    Ddl ddl = RandomDdlGenerator.builder().build().generate();
    createAndPopulate(ddl, /* numBatches = */
    100);
    // Print the Ddl
    ddl.prettyPrint();
    // Get all the tables names from the random Ddl
    List<String> tableNames = ddl.allTables().stream().map(t -> t.name()).collect(Collectors.toList());
    // Select a random number of random tables from the Ddl to export
    Random rand = new Random();
    Collections.shuffle(tableNames, rand);
    int n = rand.nextInt(tableNames.size());
    // Ensure n != 0, so that tables are actually exported
    while (n == 0) {
        n = rand.nextInt(tableNames.size());
    }
    List<String> randomExportTables = tableNames.stream().limit(n).collect(Collectors.toList());
    // Export and import the selected tables, along with any necessary related tables
    spannerServer.createDatabase(destDbPrefix + chkptSeven, Collections.emptyList());
    exportAndImportDb(sourceDb, destDbPrefix + chkptSeven, chkptSeven, String.join(",", randomExportTables), /* relatedTables =*/
    true, exportPipeline, importPipeline);
    // Compare the tables in the ddl to ensure all original tables were re-created during the import
    Collections.sort(tableNames);
    compareExpectedTables(destDbPrefix + chkptSeven, tableNames);
    // Get ALL the exported tables by calling the getFilteredTables() helper
    List<String> filteredTables = getFilteredTables(ddl, randomExportTables).stream().map(t -> t.name()).collect(Collectors.toList());
    List<String> unselectedTables = tableNames.stream().distinct().filter(t -> !filteredTables.contains(t)).collect(Collectors.toList());
    // Check to see selected tables exported with data and and unselected tables did not
    compareExpectedTableRows(destDbPrefix + chkptSeven, filteredTables, unselectedTables);
}