Examples with TableId io.debezium.relational.TableId used on opensource projects

Example 31 with TableId

use of io.debezium.relational.TableId in project debezium by debezium.

the class MySqlDdlParser method parseCreateTable.

protected void parseCreateTable(Marker start) {
    tokens.canConsume("TEMPORARY");
    tokens.consume("TABLE");
    boolean onlyIfNotExists = tokens.canConsume("IF", "NOT", "EXISTS");
    TableId tableId = parseQualifiedTableName(start);
    if (tokens.canConsume("LIKE")) {
        TableId originalId = parseQualifiedTableName(start);
        Table original = databaseTables.forTable(originalId);
        if (original != null) {
            databaseTables.overwriteTable(tableId, original.columns(), original.primaryKeyColumnNames(), original.defaultCharsetName());
        }
        consumeRemainingStatement(start);
        signalCreateTable(tableId, start);
        debugParsed(start);
        return;
    }
    if (onlyIfNotExists && databaseTables.forTable(tableId) != null) {
        // The table does exist, so we should do nothing ...
        consumeRemainingStatement(start);
        signalCreateTable(tableId, start);
        debugParsed(start);
        return;
    }
    TableEditor table = databaseTables.editOrCreateTable(tableId);
    // create_definition ...
    if (tokens.matches('('))
        parseCreateDefinitionList(start, table);
    // table_options ...
    parseTableOptions(start, table);
    // partition_options ...
    if (tokens.matches("PARTITION")) {
        parsePartitionOptions(start, table);
    }
    // select_statement
    if (tokens.canConsume("AS") || tokens.canConsume("IGNORE", "AS") || tokens.canConsume("REPLACE", "AS")) {
        parseAsSelectStatement(start, table);
    }
    // Make sure that the table's character set has been set ...
    if (!table.hasDefaultCharsetName()) {
        table.setDefaultCharsetName(currentDatabaseCharset());
    }
    // Update the table definition ...
    databaseTables.overwriteTable(table.create());
    signalCreateTable(tableId, start);
    debugParsed(start);
}

Example 32 with TableId

use of io.debezium.relational.TableId in project debezium by debezium.

the class RecordsSnapshotProducer method takeSnapshot.

private void takeSnapshot(BlockingConsumer<ChangeEvent> consumer) {
    long snapshotStart = clock().currentTimeInMillis();
    Connection jdbcConnection = null;
    try (PostgresConnection connection = taskContext.createConnection()) {
        jdbcConnection = connection.connection();
        String lineSeparator = System.lineSeparator();
        logger.info("Step 0: disabling autocommit");
        connection.setAutoCommit(false);
        long lockTimeoutMillis = taskContext.config().snapshotLockTimeoutMillis();
        logger.info("Step 1: starting transaction and refreshing the DB schemas for database '{}' and user '{}'", connection.database(), connection.username());
        // we're using the same isolation level that pg_backup uses
        StringBuilder statements = new StringBuilder("SET TRANSACTION ISOLATION LEVEL SERIALIZABLE, READ ONLY, DEFERRABLE;");
        connection.executeWithoutCommitting(statements.toString());
        statements.delete(0, statements.length());
        // next refresh the schema which will load all the tables taking the filters into account
        PostgresSchema schema = schema();
        schema.refresh(connection, false);
        logger.info("Step 2: locking each of the database tables, waiting a maximum of '{}' seconds for each lock", lockTimeoutMillis / 1000d);
        statements.append("SET lock_timeout = ").append(lockTimeoutMillis).append(";").append(lineSeparator);
        // we're locking in SHARE UPDATE EXCLUSIVE MODE to avoid concurrent schema changes while we're taking the snapshot
        // this does not prevent writes to the table, but prevents changes to the table's schema....
        // DBZ-298 Quoting name in case it has been quoted originally; it doesn't do harm if it hasn't been quoted
        schema.tables().forEach(tableId -> statements.append("LOCK TABLE ").append(tableId.toDoubleQuotedString()).append(" IN SHARE UPDATE EXCLUSIVE MODE;").append(lineSeparator));
        connection.executeWithoutCommitting(statements.toString());
        // now that we have the locks, refresh the schema
        schema.refresh(connection, false);
        // get the current position in the log, from which we'll continue streaming once the snapshot it finished
        // If rows are being inserted while we're doing the snapshot, the xlog pos should increase and so when
        // we start streaming, we should get back those changes
        long xlogStart = connection.currentXLogLocation();
        long txId = connection.currentTransactionId().longValue();
        logger.info("\t read xlogStart at '{}' from transaction '{}'", ReplicationConnection.format(xlogStart), txId);
        // and mark the start of the snapshot
        sourceInfo.startSnapshot();
        sourceInfo.update(xlogStart, clock().currentTimeInMicros(), txId);
        logger.info("Step 3: reading and exporting the contents of each table");
        AtomicInteger rowsCounter = new AtomicInteger(0);
        final Map<TableId, String> selectOverrides = getSnapshotSelectOverridesByTable();
        for (TableId tableId : schema.tables()) {
            if (schema.isFilteredOut(tableId)) {
                logger.info("\t table '{}' is filtered out, ignoring", tableId);
                continue;
            }
            long exportStart = clock().currentTimeInMillis();
            logger.info("\t exporting data from table '{}'", tableId);
            try {
                // DBZ-298 Quoting name in case it has been quoted originally; it doesn't do harm if it hasn't been quoted
                final String selectStatement = selectOverrides.getOrDefault(tableId, "SELECT * FROM " + tableId.toDoubleQuotedString());
                logger.info("For table '{}' using select statement: '{}'", tableId, selectStatement);
                connection.queryWithBlockingConsumer(selectStatement, this::readTableStatement, rs -> readTable(tableId, rs, consumer, rowsCounter));
                logger.info("\t finished exporting '{}' records for '{}'; total duration '{}'", rowsCounter.get(), tableId, Strings.duration(clock().currentTimeInMillis() - exportStart));
                rowsCounter.set(0);
            } catch (SQLException e) {
                throw new ConnectException(e);
            }
        }
        // finally commit the transaction to release all the locks...
        logger.info("Step 4: committing transaction '{}'", txId);
        jdbcConnection.commit();
        SourceRecord currentRecord = this.currentRecord.get();
        if (currentRecord != null) {
            // process and send the last record after marking it as such
            logger.info("Step 5: sending the last snapshot record");
            sourceInfo.markLastSnapshotRecord();
            this.currentRecord.set(new SourceRecord(currentRecord.sourcePartition(), sourceInfo.offset(), currentRecord.topic(), currentRecord.kafkaPartition(), currentRecord.keySchema(), currentRecord.key(), currentRecord.valueSchema(), currentRecord.value()));
            sendCurrentRecord(consumer);
        }
        // and complete the snapshot
        sourceInfo.completeSnapshot();
        logger.info("Snapshot completed in '{}'", Strings.duration(clock().currentTimeInMillis() - snapshotStart));
    } catch (SQLException e) {
        rollbackTransaction(jdbcConnection);
        throw new ConnectException(e);
    } catch (InterruptedException e) {
        Thread.interrupted();
        rollbackTransaction(jdbcConnection);
        logger.warn("Snapshot aborted after '{}'", Strings.duration(clock().currentTimeInMillis() - snapshotStart));
    }
}

Example 33 with TableId

use of io.debezium.relational.TableId in project debezium by debezium.

the class AbstractRecordsProducerTest method tableIdFromInsertStmt.

protected static TableId tableIdFromInsertStmt(String statement) {
    Matcher matcher = INSERT_TABLE_MATCHING_PATTERN.matcher(statement);
    assertTrue("Extraction of table name from insert statement failed: " + statement, matcher.matches());
    TableId id = TableId.parse(matcher.group(1), false);
    if (id.schema() == null) {
        id = new TableId(id.catalog(), "public", id.table());
    }
    return id;
}

Example 34 with TableId

use of io.debezium.relational.TableId in project debezium by debezium.

the class MySqlSchemaTest method assertTableIncluded.

protected void assertTableIncluded(String fullyQualifiedTableName) {
    TableId tableId = TableId.parse(fullyQualifiedTableName);
    assertThat(mysql.tables().forTable(tableId)).isNotNull();
    TableSchema tableSchema = mysql.schemaFor(tableId);
    assertThat(tableSchema).isNotNull();
    assertThat(tableSchema.keySchema().name()).isEqualTo(SchemaNameAdjuster.validFullname(SERVER_NAME + "." + fullyQualifiedTableName + ".Key"));
    assertThat(tableSchema.valueSchema().name()).isEqualTo(SchemaNameAdjuster.validFullname(SERVER_NAME + "." + fullyQualifiedTableName + ".Value"));
}

Example 35 with TableId

use of io.debezium.relational.TableId in project debezium by debezium.

the class MySqlSchemaTest method assertTableExcluded.

protected void assertTableExcluded(String fullyQualifiedTableName) {
    TableId tableId = TableId.parse(fullyQualifiedTableName);
    assertThat(mysql.tables().forTable(tableId)).isNull();
    assertThat(mysql.schemaFor(tableId)).isNull();
}