Examples with Tables io.debezium.relational.Tables used on opensource projects

Example 1 with Tables

use of io.debezium.relational.Tables in project debezium by debezium.

the class AbstractDatabaseHistoryTest method beforeEach.

@Before
public void beforeEach() {
    parser = new DdlParserSql2003();
    tables = new Tables();
    t0 = new Tables();
    t1 = new Tables();
    t2 = new Tables();
    t3 = new Tables();
    t4 = new Tables();
    all = new Tables();
    source1 = server("abc");
    source2 = server("xyz");
    history = createHistory();
}

Example 2 with Tables

use of io.debezium.relational.Tables in project debezium by debezium.

the class KafkaDatabaseHistoryTest method testHistoryTopicContent.

private void testHistoryTopicContent(boolean skipUnparseableDDL) {
    // Start up the history ...
    Configuration config = Configuration.create().with(KafkaDatabaseHistory.BOOTSTRAP_SERVERS, kafka.brokerList()).with(KafkaDatabaseHistory.TOPIC, topicName).with(DatabaseHistory.NAME, "my-db-history").with(KafkaDatabaseHistory.RECOVERY_POLL_INTERVAL_MS, 500).with(KafkaDatabaseHistory.consumerConfigPropertyName(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG), 100).with(KafkaDatabaseHistory.consumerConfigPropertyName(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), 50000).with(KafkaDatabaseHistory.SKIP_UNPARSEABLE_DDL_STATEMENTS, skipUnparseableDDL).build();
    history.configure(config, null);
    history.start();
    // Should be able to call start more than once ...
    history.start();
    history.initializeStorage();
    // Calling it another time to ensure we can work with the DB history topic already existing
    history.initializeStorage();
    DdlParser recoveryParser = new DdlParserSql2003();
    DdlParser ddlParser = new DdlParserSql2003();
    // recover does this, so we need to as well
    ddlParser.setCurrentSchema("db1");
    Tables tables1 = new Tables();
    Tables tables2 = new Tables();
    Tables tables3 = new Tables();
    // Recover from the very beginning ...
    setLogPosition(0);
    history.recover(source, position, tables1, recoveryParser);
    // There should have been nothing to recover ...
    assertThat(tables1.size()).isEqualTo(0);
    // Now record schema changes, which writes out to kafka but doesn't actually change the Tables ...
    setLogPosition(10);
    ddl = "CREATE TABLE foo ( name VARCHAR(255) NOT NULL PRIMARY KEY); \n" + "CREATE TABLE customers ( id INTEGER NOT NULL PRIMARY KEY, name VARCHAR(100) NOT NULL ); \n" + "CREATE TABLE products ( productId INTEGER NOT NULL PRIMARY KEY, desc VARCHAR(255) NOT NULL); \n";
    history.record(source, position, "db1", ddl);
    // Parse the DDL statement 3x and each time update a different Tables object ...
    ddlParser.parse(ddl, tables1);
    assertThat(tables1.size()).isEqualTo(3);
    ddlParser.parse(ddl, tables2);
    assertThat(tables2.size()).isEqualTo(3);
    ddlParser.parse(ddl, tables3);
    assertThat(tables3.size()).isEqualTo(3);
    // Record a drop statement and parse it for 2 of our 3 Tables...
    setLogPosition(39);
    ddl = "DROP TABLE foo;";
    history.record(source, position, "db1", ddl);
    ddlParser.parse(ddl, tables2);
    assertThat(tables2.size()).isEqualTo(2);
    ddlParser.parse(ddl, tables3);
    assertThat(tables3.size()).isEqualTo(2);
    // Record another DDL statement and parse it for 1 of our 3 Tables...
    setLogPosition(10003);
    ddl = "CREATE TABLE suppliers ( supplierId INTEGER NOT NULL PRIMARY KEY, name VARCHAR(255) NOT NULL);";
    history.record(source, position, "db1", ddl);
    ddlParser.parse(ddl, tables3);
    assertThat(tables3.size()).isEqualTo(3);
    // Stop the history (which should stop the producer) ...
    history.stop();
    history = new KafkaDatabaseHistory();
    history.configure(config, null);
    // no need to start
    // Recover from the very beginning to just past the first change ...
    Tables recoveredTables = new Tables();
    setLogPosition(15);
    history.recover(source, position, recoveredTables, recoveryParser);
    assertThat(recoveredTables).isEqualTo(tables1);
    // Recover from the very beginning to just past the second change ...
    recoveredTables = new Tables();
    setLogPosition(50);
    history.recover(source, position, recoveredTables, recoveryParser);
    assertThat(recoveredTables).isEqualTo(tables2);
    // Recover from the very beginning to just past the third change ...
    recoveredTables = new Tables();
    setLogPosition(10010);
    history.recover(source, position, recoveredTables, recoveryParser);
    assertThat(recoveredTables).isEqualTo(tables3);
    // Recover from the very beginning to way past the third change ...
    recoveredTables = new Tables();
    setLogPosition(100000010);
    history.recover(source, position, recoveredTables, recoveryParser);
    assertThat(recoveredTables).isEqualTo(tables3);
}

Example 3 with Tables

use of io.debezium.relational.Tables in project debezium by debezium.

the class PostgresSchema method refresh.

/**
 * Refreshes this schema's content for a particular table
 *
 * @param connection a {@link JdbcConnection} instance, never {@code null}
 * @param tableId the table identifier; may not be null
 * @throws SQLException if there is a problem refreshing the schema from the database server
 */
protected void refresh(PostgresConnection connection, TableId tableId) throws SQLException {
    Tables temp = new Tables();
    Tables.TableNameFilter tableNameFilter = Tables.filterFor(Predicate.isEqual(tableId));
    connection.readSchema(temp, null, null, tableNameFilter, null, true);
    // we expect the refreshed table to be there
    assert temp.size() == 1;
    // overwrite (add or update) or views of the tables
    tables.overwriteTable(temp.forTable(tableId));
    // and refresh the schema
    refreshSchema(tableId);
}

Example 4 with Tables

use of io.debezium.relational.Tables in project debezium by debezium.

the class MySqlDdlParserTest method beforeEach.

@Before
public void beforeEach() {
    parser = new MySqlDdlParser();
    tables = new Tables();
    listener = new SimpleDdlParserListener();
    parser.addListener(listener);
}

Example 5 with Tables

use of io.debezium.relational.Tables in project debezium by debezium.

the class MetadataIT method shouldLoadMetadataViaJdbc.

/**
 * Loads the {@link Tables} definition by reading JDBC metadata. Note that some characteristics, such as whether columns
 * are generated, are not exposed through JDBC (unlike when reading DDL).
 * @throws SQLException if there's an error
 */
@Test
public void shouldLoadMetadataViaJdbc() throws SQLException {
    final UniqueDatabase DATABASE = new UniqueDatabase("readbinlog_it", "readbinlog_test");
    DATABASE.createAndInitialize();
    try (MySQLConnection conn = MySQLConnection.forTestDatabase(DATABASE.getDatabaseName())) {
        conn.connect();
        // Set up the table as one transaction and wait to see the events ...
        conn.execute("DROP TABLE IF EXISTS person", "DROP TABLE IF EXISTS product", "DROP TABLE IF EXISTS purchased");
        conn.execute("CREATE TABLE person (" + "  name VARCHAR(255) primary key," + "  birthdate DATE NULL," + "  age INTEGER NULL DEFAULT 10," + "  salary DECIMAL(5,2)," + "  bitStr BIT(18)" + ")");
        conn.execute("SELECT * FROM person");
        Tables tables = new Tables();
        conn.readSchema(tables, DATABASE.getDatabaseName(), null, null, null, true);
        // System.out.println(tables);
        assertThat(tables.size()).isEqualTo(1);
        Table person = tables.forTable(DATABASE.getDatabaseName(), null, "person");
        assertThat(person).isNotNull();
        assertThat(person.filterColumns(col -> col.isAutoIncremented())).isEmpty();
        assertThat(person.primaryKeyColumnNames()).containsOnly("name");
        assertThat(person.columnNames()).containsExactly("name", "birthdate", "age", "salary", "bitStr");
        assertThat(person.columnWithName("name").name()).isEqualTo("name");
        assertThat(person.columnWithName("name").typeName()).isEqualTo("VARCHAR");
        assertThat(person.columnWithName("name").jdbcType()).isEqualTo(Types.VARCHAR);
        assertThat(person.columnWithName("name").length()).isEqualTo(255);
        assertThat(person.columnWithName("name").scale()).isEqualTo(0);
        assertThat(person.columnWithName("name").position()).isEqualTo(1);
        assertThat(person.columnWithName("name").isAutoIncremented()).isFalse();
        assertThat(person.columnWithName("name").isGenerated()).isFalse();
        assertThat(person.columnWithName("name").isOptional()).isFalse();
        assertThat(person.columnWithName("birthdate").name()).isEqualTo("birthdate");
        assertThat(person.columnWithName("birthdate").typeName()).isEqualTo("DATE");
        assertThat(person.columnWithName("birthdate").jdbcType()).isEqualTo(Types.DATE);
        assertThat(person.columnWithName("birthdate").length()).isEqualTo(10);
        assertThat(person.columnWithName("birthdate").scale()).isEqualTo(0);
        assertThat(person.columnWithName("birthdate").position()).isEqualTo(2);
        assertThat(person.columnWithName("birthdate").isAutoIncremented()).isFalse();
        assertThat(person.columnWithName("birthdate").isGenerated()).isFalse();
        assertThat(person.columnWithName("birthdate").isOptional()).isTrue();
        assertThat(person.columnWithName("age").name()).isEqualTo("age");
        assertThat(person.columnWithName("age").typeName()).isEqualTo("INT");
        assertThat(person.columnWithName("age").jdbcType()).isEqualTo(Types.INTEGER);
        assertThat(person.columnWithName("age").length()).isEqualTo(10);
        assertThat(person.columnWithName("age").scale()).isEqualTo(0);
        assertThat(person.columnWithName("age").position()).isEqualTo(3);
        assertThat(person.columnWithName("age").isAutoIncremented()).isFalse();
        assertThat(person.columnWithName("age").isGenerated()).isFalse();
        assertThat(person.columnWithName("age").isOptional()).isTrue();
        assertThat(person.columnWithName("salary").name()).isEqualTo("salary");
        assertThat(person.columnWithName("salary").typeName()).isEqualTo("DECIMAL");
        assertThat(person.columnWithName("salary").jdbcType()).isEqualTo(Types.DECIMAL);
        assertThat(person.columnWithName("salary").length()).isEqualTo(5);
        assertThat(person.columnWithName("salary").scale()).isEqualTo(2);
        assertThat(person.columnWithName("salary").position()).isEqualTo(4);
        assertThat(person.columnWithName("salary").isAutoIncremented()).isFalse();
        assertThat(person.columnWithName("salary").isGenerated()).isFalse();
        assertThat(person.columnWithName("salary").isOptional()).isTrue();
        assertThat(person.columnWithName("bitStr").name()).isEqualTo("bitStr");
        assertThat(person.columnWithName("bitStr").typeName()).isEqualTo("BIT");
        assertThat(person.columnWithName("bitStr").jdbcType()).isEqualTo(Types.BIT);
        assertThat(person.columnWithName("bitStr").length()).isEqualTo(18);
        assertThat(person.columnWithName("bitStr").scale()).isEqualTo(0);
        assertThat(person.columnWithName("bitStr").position()).isEqualTo(5);
        assertThat(person.columnWithName("bitStr").isAutoIncremented()).isFalse();
        assertThat(person.columnWithName("bitStr").isGenerated()).isFalse();
        assertThat(person.columnWithName("bitStr").isOptional()).isTrue();
        conn.execute("CREATE TABLE product (" + "  id INT NOT NULL AUTO_INCREMENT," + "  createdByDate DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP," + "  modifiedDate DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," + "  PRIMARY KEY(id)" + ")");
        conn.execute("SELECT * FROM product");
        tables = new Tables();
        conn.readSchema(tables, DATABASE.getDatabaseName(), null, null, null, true);
        // System.out.println(tables);
        assertThat(tables.size()).isEqualTo(2);
        Table product = tables.forTable(DATABASE.getDatabaseName(), null, "product");
        assertThat(product).isNotNull();
        assertThat(product.filterColumnNames(Column::isAutoIncremented)).containsOnly("id");
        assertThat(product.primaryKeyColumnNames()).containsOnly("id");
        assertThat(product.columnNames()).containsExactly("id", "createdByDate", "modifiedDate");
        assertThat(product.columnWithName("id").name()).isEqualTo("id");
        assertThat(product.columnWithName("id").typeName()).isEqualTo("INT");
        assertThat(product.columnWithName("id").jdbcType()).isEqualTo(Types.INTEGER);
        assertThat(product.columnWithName("id").length()).isEqualTo(10);
        assertThat(product.columnWithName("id").scale()).isEqualTo(0);
        assertThat(product.columnWithName("id").position()).isEqualTo(1);
        assertThat(product.columnWithName("id").isAutoIncremented()).isTrue();
        assertThat(product.columnWithName("id").isGenerated()).isFalse();
        assertThat(product.columnWithName("id").isOptional()).isFalse();
        assertThat(product.columnWithName("createdByDate").name()).isEqualTo("createdByDate");
        assertThat(product.columnWithName("createdByDate").typeName()).isEqualTo("DATETIME");
        assertThat(product.columnWithName("createdByDate").jdbcType()).isEqualTo(Types.TIMESTAMP);
        assertThat(product.columnWithName("createdByDate").length()).isEqualTo(19);
        assertThat(product.columnWithName("createdByDate").scale()).isEqualTo(0);
        assertThat(product.columnWithName("createdByDate").position()).isEqualTo(2);
        assertThat(product.columnWithName("createdByDate").isAutoIncremented()).isFalse();
        assertThat(product.columnWithName("createdByDate").isGenerated()).isFalse();
        assertThat(product.columnWithName("createdByDate").isOptional()).isFalse();
        assertThat(product.columnWithName("modifiedDate").name()).isEqualTo("modifiedDate");
        assertThat(product.columnWithName("modifiedDate").typeName()).isEqualTo("DATETIME");
        assertThat(product.columnWithName("modifiedDate").jdbcType()).isEqualTo(Types.TIMESTAMP);
        assertThat(product.columnWithName("modifiedDate").length()).isEqualTo(19);
        assertThat(product.columnWithName("modifiedDate").scale()).isEqualTo(0);
        assertThat(product.columnWithName("modifiedDate").position()).isEqualTo(3);
        assertThat(product.columnWithName("modifiedDate").isAutoIncremented()).isFalse();
        assertThat(product.columnWithName("modifiedDate").isGenerated()).isFalse();
        assertThat(product.columnWithName("modifiedDate").isOptional()).isFalse();
        conn.execute("CREATE TABLE purchased (" + "  purchaser VARCHAR(255) NOT NULL," + "  productId INT NOT NULL," + "  purchaseDate DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP," + "  PRIMARY KEY(productId,purchaser)" + ")");
        conn.execute("SELECT * FROM purchased");
        tables = new Tables();
        conn.readSchema(tables, DATABASE.getDatabaseName(), null, null, null, true);
        // System.out.println(tables);
        assertThat(tables.size()).isEqualTo(3);
        Table purchased = tables.forTable(DATABASE.getDatabaseName(), null, "purchased");
        assertThat(purchased).isNotNull();
        assertThat(person.filterColumns(col -> col.isAutoIncremented())).isEmpty();
        assertThat(purchased.primaryKeyColumnNames()).containsOnly("productId", "purchaser");
        assertThat(purchased.columnNames()).containsExactly("purchaser", "productId", "purchaseDate");
        assertThat(purchased.columnWithName("purchaser").name()).isEqualTo("purchaser");
        assertThat(purchased.columnWithName("purchaser").typeName()).isEqualTo("VARCHAR");
        assertThat(purchased.columnWithName("purchaser").jdbcType()).isEqualTo(Types.VARCHAR);
        assertThat(purchased.columnWithName("purchaser").length()).isEqualTo(255);
        assertThat(purchased.columnWithName("purchaser").scale()).isEqualTo(0);
        assertThat(purchased.columnWithName("purchaser").position()).isEqualTo(1);
        assertThat(purchased.columnWithName("purchaser").isAutoIncremented()).isFalse();
        assertThat(purchased.columnWithName("purchaser").isGenerated()).isFalse();
        assertThat(purchased.columnWithName("purchaser").isOptional()).isFalse();
        assertThat(purchased.columnWithName("productId").name()).isEqualTo("productId");
        assertThat(purchased.columnWithName("productId").typeName()).isEqualTo("INT");
        assertThat(purchased.columnWithName("productId").jdbcType()).isEqualTo(Types.INTEGER);
        assertThat(purchased.columnWithName("productId").length()).isEqualTo(10);
        assertThat(purchased.columnWithName("productId").scale()).isEqualTo(0);
        assertThat(purchased.columnWithName("productId").position()).isEqualTo(2);
        assertThat(purchased.columnWithName("productId").isAutoIncremented()).isFalse();
        assertThat(purchased.columnWithName("productId").isGenerated()).isFalse();
        assertThat(purchased.columnWithName("productId").isOptional()).isFalse();
        assertThat(purchased.columnWithName("purchaseDate").name()).isEqualTo("purchaseDate");
        assertThat(purchased.columnWithName("purchaseDate").typeName()).isEqualTo("DATETIME");
        assertThat(purchased.columnWithName("purchaseDate").jdbcType()).isEqualTo(Types.TIMESTAMP);
        assertThat(purchased.columnWithName("purchaseDate").length()).isEqualTo(19);
        assertThat(purchased.columnWithName("purchaseDate").scale()).isEqualTo(0);
        assertThat(purchased.columnWithName("purchaseDate").position()).isEqualTo(3);
        assertThat(purchased.columnWithName("purchaseDate").isAutoIncremented()).isFalse();
        assertThat(purchased.columnWithName("purchaseDate").isGenerated()).isFalse();
        assertThat(purchased.columnWithName("purchaseDate").isOptional()).isFalse();
    }
}