Examples with Table com.google.cloud.teleport.v2.templates.spanner.ddl.Table used on opensource projects

Example 26 with Table

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class ShadowTableCreatorTest method canConstructShadowTableForOracle.

@Test
public void canConstructShadowTableForOracle() {
    Ddl testDdl = ProcessInformationSchemaTest.getTestDdl();
    ShadowTableCreator shadowTableCreator = new ShadowTableCreator("oracle", "shadow_");
    Table shadowTable = shadowTableCreator.constructShadowTable(testDdl, "Users_interleaved");
    /* Verify
     * (1) name of shadow table
     * (2) primary keys columns are same as data tables
     * (3) Has oracle sequence information column in addition to primary keys columns
     */
    assertEquals(shadowTable.name(), "shadow_Users_interleaved");
    assertThat(shadowTable.primaryKeys(), is(testDdl.table("Users_interleaved").primaryKeys()));
    Set<String> columns = shadowTable.columns().stream().map(c -> c.name()).collect(Collectors.toSet());
    Set<String> expectedColumns = testDdl.table("Users_interleaved").primaryKeys().stream().map(c -> c.name()).collect(Collectors.toSet());
    expectedColumns.add("timestamp");
    expectedColumns.add("scn");
    assertThat(columns, is(expectedColumns));
}

Example 27 with Table

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class SpannerStreamingWriteIntegrationTest method canWriteDisorderedAndInterleavedChangeEvents.

@Test
public void canWriteDisorderedAndInterleavedChangeEvents() throws Exception {
    JSONObject json1 = getChangeEventForTable1("1", "334", "INSERT", "1");
    JSONObject json2 = getChangeEvent("Table1_interleaved", "INSERT", "2");
    json2.put("id", "1");
    json2.put("id2", "1");
    json2.put("data2", "32");
    /* The order of event processing cannot be predicted or controlled in
     * Test pipelines. The order in the Arrays below does not mean the events
     * are processed in that order.
     * As long as atleast 1 change event for the interleaved table is after the
     * parent table, this test will be successful.
     * Hence change event for interleaved table is repeated multiple times.
     * This also mimics the retry behavior during interleaved tables handling.
     */
    PCollection<FailsafeElement<String, String>> jsonRecords = testPipeline.apply(Create.of(Arrays.asList(FailsafeElement.of(json2.toString(), json2.toString()), FailsafeElement.of(json2.toString(), json2.toString()), FailsafeElement.of(json1.toString(), json1.toString()), FailsafeElement.of(json2.toString(), json2.toString()), FailsafeElement.of(json2.toString(), json2.toString()), FailsafeElement.of(json2.toString(), json2.toString()), FailsafeElement.of(json2.toString(), json2.toString()), FailsafeElement.of(json2.toString(), json2.toString()))).withCoder(FailsafeElementCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())));
    constructAndRunPipeline(jsonRecords);
    verifyRecordCountinTable("Table1", 1);
    verifyRecordCountinTable("Table1_interleaved", 1);
}

Example 28 with Table

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class BigQueryToElasticsearch method run.

/**
 * Runs the pipeline with the supplied options.
 *
 * @param options The execution parameters to the pipeline.
 * @return The result of the pipeline execution.
 */
private static PipelineResult run(BigQueryToElasticsearchOptions options) {
    // Create the pipeline.
    Pipeline pipeline = Pipeline.create(options);
    /*
     * Steps: 1) Read records from BigQuery via BigQueryIO.
     *        2) Create json string from Table Row.
     *        3) Write records to Elasticsearch.
     *
     *
     * Step #1: Read from BigQuery. If a query is provided then it is used to get the TableRows.
     */
    pipeline.apply("ReadFromBigQuery", ReadBigQuery.newBuilder().setOptions(options.as(BigQueryToElasticsearchOptions.class)).build()).apply("TableRowsToJsonDocument", ParDo.of(new TableRowToJsonFn())).apply("WriteToElasticsearch", WriteToElasticsearch.newBuilder().setOptions(options.as(BigQueryToElasticsearchOptions.class)).build());
    return pipeline.run();
}

Example 29 with Table

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class DatastreamToDMLTest method testGetPostgresTableName.

/**
 * Test whether {@link DatastreamToPostgresDML#getTargetTableName(row)} converts the Oracle table
 * into the correct Postgres table.
 */
@Test
public void testGetPostgresTableName() {
    DatastreamToDML datastreamToDML = DatastreamToPostgresDML.of(null);
    JsonNode rowObj = this.getRowObj();
    DatastreamRow row = DatastreamRow.of(rowObj);
    String expectedTableName = "my_table$name";
    String tableName = datastreamToDML.getTargetTableName(row);
    assertEquals(expectedTableName, tableName);
}

Example 30 with Table

use of com.google.cloud.teleport.v2.templates.spanner.ddl.Table in project DataflowTemplates by GoogleCloudPlatform.

the class DatastreamToDML method convertJsonToDmlInfo.

public DmlInfo convertJsonToDmlInfo(JsonNode rowObj, String failsafeValue) {
    DatastreamRow row = DatastreamRow.of(rowObj);
    try {
        // Oracle uses upper case while Postgres uses all lowercase.
        // We lowercase the values of these metadata fields to align with
        // our schema conversion rules.
        String catalogName = this.getTargetCatalogName(row);
        String schemaName = this.getTargetSchemaName(row);
        String tableName = this.getTargetTableName(row);
        Map<String, String> tableSchema = this.getTableSchema(catalogName, schemaName, tableName);
        if (tableSchema.isEmpty()) {
            // If the table DNE we return null (NOOP).
            return null;
        }
        List<String> primaryKeys = this.getPrimaryKeys(catalogName, schemaName, tableName, rowObj);
        List<String> orderByFields = row.getSortFields();
        List<String> primaryKeyValues = getFieldValues(rowObj, primaryKeys, tableSchema);
        List<String> orderByValues = getFieldValues(rowObj, orderByFields, tableSchema);
        String dmlSqlTemplate = getDmlTemplate(rowObj, primaryKeys);
        Map<String, String> sqlTemplateValues = getSqlTemplateValues(rowObj, catalogName, schemaName, tableName, primaryKeys, tableSchema);
        String dmlSql = StringSubstitutor.replace(dmlSqlTemplate, sqlTemplateValues, "{", "}");
        return DmlInfo.of(failsafeValue, dmlSql, schemaName, tableName, primaryKeys, orderByFields, primaryKeyValues, orderByValues);
    } catch (DeletedWithoutPrimaryKey e) {
        LOG.error("CDC Error: {} :: {}", rowObj.toString(), e.toString());
        return null;
    } catch (Exception e) {
        // TODO(dhercher): Consider raising an error and pushing to DLQ
        LOG.error("Value Error: {} :: {}", rowObj.toString(), e.toString());
        return null;
    }
}