Examples with JsonDebeziumDeserializationSchema com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema used on opensource projects

Example 1 with JsonDebeziumDeserializationSchema

use of com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema in project dlink by DataLinkDC.

the class FlinkCDCMergeBuilder method buildMySqlCDC.

public static void buildMySqlCDC(StreamExecutionEnvironment env, FlinkCDCConfig config) {
    if (Asserts.isNotNull(config.getParallelism())) {
        env.setParallelism(config.getParallelism());
    }
    if (Asserts.isNotNull(config.getCheckpoint())) {
        env.enableCheckpointing(config.getCheckpoint());
    }
    MySqlSourceBuilder<String> sourceBuilder = MySqlSource.<String>builder().hostname(config.getHostname()).port(config.getPort()).username(config.getUsername()).password(config.getPassword());
    if (Asserts.isNotNull(config.getDatabase()) && config.getDatabase().size() > 0) {
        sourceBuilder.databaseList(config.getDatabase().toArray(new String[0]));
    }
    if (Asserts.isNotNull(config.getTable()) && config.getTable().size() > 0) {
        sourceBuilder.tableList(config.getTable().toArray(new String[0]));
    }
    MySqlSourceBuilder<String> builder = sourceBuilder.deserializer(new JsonDebeziumDeserializationSchema());
    if (Asserts.isNotNullString(config.getStartupMode())) {
        switch(config.getStartupMode().toUpperCase()) {
            case "INITIAL":
                builder.startupOptions(StartupOptions.initial());
                break;
            case "EARLIEST":
                builder.startupOptions(StartupOptions.earliest());
                break;
            case "LATEST":
                builder.startupOptions(StartupOptions.latest());
                break;
            default:
                builder.startupOptions(StartupOptions.latest());
        }
    } else {
        builder.startupOptions(StartupOptions.latest());
    }
    MySqlSource<String> sourceFunction = builder.build();
    DataStreamSource<String> streamSource = env.fromSource(sourceFunction, WatermarkStrategy.noWatermarks(), "MySQL Source");
    streamSource.addSink(getKafkaProducer(config.getBrokers(), config.getTopic()));
}

Example 2 with JsonDebeziumDeserializationSchema

use of com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema in project flink-cdc-connectors by ververica.

the class LegacyMySqlSourceITCase method testConsumingAllEventsWithJsonFormat.

private void testConsumingAllEventsWithJsonFormat(Boolean includeSchema, Map<String, Object> customConverterConfigs, String expectedFile) throws Exception {
    fullTypesDatabase.createAndInitialize();
    JsonDebeziumDeserializationSchema schema = customConverterConfigs == null ? new JsonDebeziumDeserializationSchema(includeSchema) : new JsonDebeziumDeserializationSchema(includeSchema, customConverterConfigs);
    SourceFunction<String> sourceFunction = MySqlSource.<String>builder().hostname(MYSQL_CONTAINER.getHost()).port(MYSQL_CONTAINER.getDatabasePort()).databaseList(fullTypesDatabase.getDatabaseName()).username(fullTypesDatabase.getUsername()).password(fullTypesDatabase.getPassword()).deserializer(schema).build();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(1000);
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env, EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build());
    final JSONObject expected = JSONObject.parseObject(readLines(expectedFile), JSONObject.class);
    JSONObject expectSnapshot = expected.getJSONObject("expected_snapshot");
    DataStreamSource<String> source = env.addSource(sourceFunction);
    tEnv.createTemporaryView("full_types", source);
    TableResult result = tEnv.executeSql("SELECT * FROM full_types");
    // check the snapshot result
    CloseableIterator<Row> snapshot = result.collect();
    waitForSnapshotStarted(snapshot);
    assertTrue(dataInJsonIsEquals(fetchRows(snapshot, 1).get(0).toString(), expectSnapshot.toString()));
    try (Connection connection = fullTypesDatabase.getJdbcConnection();
        Statement statement = connection.createStatement()) {
        statement.execute("UPDATE full_types SET timestamp_c = '2020-07-17 18:33:22' WHERE id=1;");
    }
    // check the binlog result
    CloseableIterator<Row> binlog = result.collect();
    JSONObject expectBinlog = expected.getJSONObject("expected_binlog");
    assertTrue(dataInJsonIsEquals(fetchRows(binlog, 1).get(0).toString(), expectBinlog.toString()));
    result.getJobClient().get().cancel().get();
}

Example 3 with JsonDebeziumDeserializationSchema

use of com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema in project flink-cdc-connectors by ververica.

the class MySqlChangeEventSourceExampleTest method testConsumingAllEvents.

@Test
@Ignore("Test ignored because it won't stop and is used for manual test")
public void testConsumingAllEvents() throws Exception {
    inventoryDatabase.createAndInitialize();
    JdbcIncrementalSource<String> mySqlChangeEventSource = new MySqlSourceBuilder().hostname(MYSQL_CONTAINER.getHost()).port(MYSQL_CONTAINER.getDatabasePort()).databaseList(inventoryDatabase.getDatabaseName()).tableList(inventoryDatabase.getDatabaseName() + ".products").username(inventoryDatabase.getUsername()).password(inventoryDatabase.getPassword()).serverId("5401-5404").deserializer(new JsonDebeziumDeserializationSchema()).includeSchemaChanges(// output the schema changes as well
    true).build();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // enable checkpoint
    env.enableCheckpointing(3000);
    // set the source parallelism to 4
    env.fromSource(mySqlChangeEventSource, WatermarkStrategy.noWatermarks(), "MySqlParallelSource").setParallelism(4).print().setParallelism(1);
    env.execute("Print MySQL Snapshot + Binlog");
}

Example 4 with JsonDebeziumDeserializationSchema

use of com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema in project dlink by DataLinkDC.

the class MysqlCDCBuilder method build.

@Override
public DataStreamSource<String> build(StreamExecutionEnvironment env) {
    String database = config.getDatabase();
    String serverId = config.getSource().get("server-id");
    String serverTimeZone = config.getSource().get("server-time-zone");
    String fetchSize = config.getSource().get("scan.snapshot.fetch.size");
    String connectTimeout = config.getSource().get("connect.timeout");
    String connectMaxRetries = config.getSource().get("connect.max-retries");
    String connectionPoolSize = config.getSource().get("connection.pool.size");
    String heartbeatInterval = config.getSource().get("heartbeat.interval");
    Properties debeziumProperties = new Properties();
    // 为部分转换添加默认值
    debeziumProperties.setProperty("bigint.unsigned.handling.mode", "long");
    debeziumProperties.setProperty("decimal.handling.mode", "string");
    for (Map.Entry<String, String> entry : config.getDebezium().entrySet()) {
        if (Asserts.isNotNullString(entry.getKey()) && Asserts.isNotNullString(entry.getValue())) {
            debeziumProperties.setProperty(entry.getKey(), entry.getValue());
        }
    }
    // 添加jdbc参数注入
    Properties jdbcProperties = new Properties();
    for (Map.Entry<String, String> entry : config.getJdbc().entrySet()) {
        if (Asserts.isNotNullString(entry.getKey()) && Asserts.isNotNullString(entry.getValue())) {
            jdbcProperties.setProperty(entry.getKey(), entry.getValue());
        }
    }
    MySqlSourceBuilder<String> sourceBuilder = MySqlSource.<String>builder().hostname(config.getHostname()).port(config.getPort()).username(config.getUsername()).password(config.getPassword());
    if (Asserts.isNotNullString(database)) {
        String[] databases = database.split(FlinkParamConstant.SPLIT);
        sourceBuilder.databaseList(databases);
    } else {
        sourceBuilder.databaseList(new String[0]);
    }
    List<String> schemaTableNameList = config.getSchemaTableNameList();
    if (Asserts.isNotNullCollection(schemaTableNameList)) {
        sourceBuilder.tableList(schemaTableNameList.toArray(new String[schemaTableNameList.size()]));
    } else {
        sourceBuilder.tableList(new String[0]);
    }
    sourceBuilder.deserializer(new JsonDebeziumDeserializationSchema());
    sourceBuilder.debeziumProperties(debeziumProperties);
    sourceBuilder.jdbcProperties(jdbcProperties);
    if (Asserts.isNotNullString(config.getStartupMode())) {
        switch(config.getStartupMode().toLowerCase()) {
            case "initial":
                sourceBuilder.startupOptions(StartupOptions.initial());
                break;
            case "latest-offset":
                sourceBuilder.startupOptions(StartupOptions.latest());
                break;
        }
    } else {
        sourceBuilder.startupOptions(StartupOptions.latest());
    }
    if (Asserts.isNotNullString(serverId)) {
        sourceBuilder.serverId(serverId);
    }
    if (Asserts.isNotNullString(serverTimeZone)) {
        sourceBuilder.serverTimeZone(serverTimeZone);
    }
    if (Asserts.isNotNullString(fetchSize)) {
        sourceBuilder.fetchSize(Integer.valueOf(fetchSize));
    }
    if (Asserts.isNotNullString(connectTimeout)) {
        sourceBuilder.connectTimeout(Duration.ofMillis(Long.valueOf(connectTimeout)));
    }
    if (Asserts.isNotNullString(connectMaxRetries)) {
        sourceBuilder.connectMaxRetries(Integer.valueOf(connectMaxRetries));
    }
    if (Asserts.isNotNullString(connectionPoolSize)) {
        sourceBuilder.connectionPoolSize(Integer.valueOf(connectionPoolSize));
    }
    if (Asserts.isNotNullString(heartbeatInterval)) {
        sourceBuilder.heartbeatInterval(Duration.ofMillis(Long.valueOf(heartbeatInterval)));
    }
    return env.fromSource(sourceBuilder.build(), WatermarkStrategy.noWatermarks(), "MySQL CDC Source");
}

Example 5 with JsonDebeziumDeserializationSchema

use of com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema in project dlink by DataLinkDC.

the class OracleCDCBuilder method build.

@Override
public DataStreamSource<String> build(StreamExecutionEnvironment env) {
    Properties properties = new Properties();
    for (Map.Entry<String, String> entry : config.getDebezium().entrySet()) {
        if (Asserts.isNotNullString(entry.getKey()) && Asserts.isNotNullString(entry.getValue())) {
            properties.setProperty(entry.getKey(), entry.getValue());
        }
    }
    OracleSource.Builder<String> sourceBuilder = OracleSource.<String>builder().hostname(config.getHostname()).port(config.getPort()).username(config.getUsername()).password(config.getPassword()).database(config.getDatabase());
    String schema = config.getSchema();
    if (Asserts.isNotNullString(schema)) {
        String[] schemas = schema.split(FlinkParamConstant.SPLIT);
        sourceBuilder.schemaList(schemas);
    } else {
        sourceBuilder.schemaList(new String[0]);
    }
    List<String> schemaTableNameList = config.getSchemaTableNameList();
    if (Asserts.isNotNullCollection(schemaTableNameList)) {
        sourceBuilder.tableList(schemaTableNameList.toArray(new String[schemaTableNameList.size()]));
    } else {
        sourceBuilder.tableList(new String[0]);
    }
    sourceBuilder.deserializer(new JsonDebeziumDeserializationSchema());
    sourceBuilder.debeziumProperties(properties);
    if (Asserts.isNotNullString(config.getStartupMode())) {
        switch(config.getStartupMode().toLowerCase()) {
            case "initial":
                sourceBuilder.startupOptions(StartupOptions.initial());
                break;
            case "latest-offset":
                sourceBuilder.startupOptions(StartupOptions.latest());
                break;
        }
    } else {
        sourceBuilder.startupOptions(StartupOptions.latest());
    }
    return env.addSource(sourceBuilder.build(), "Oracle CDC Source");
}