Examples with Table io.debezium.relational.Table used on opensource projects

Example 26 with Table

use of io.debezium.relational.Table in project debezium by debezium.

the class MySqlSchema method applyDdl.

/**
 * Apply the supplied DDL statements to this database schema and record the history. If a {@code statementConsumer} is
 * supplied, then call it for each sub-sequence of the DDL statements that all apply to the same database.
 * <p>
 * Typically DDL statements are applied using a connection to a single database, and unless the statements use fully-qualified
 * names, the DDL statements apply to this database.
 *
 * @param source the current {@link SourceInfo#partition()} and {@link SourceInfo#offset() offset} at which these changes are
 *            found; may not be null
 * @param databaseName the name of the default database under which these statements are applied; may not be null
 * @param ddlStatements the {@code ;}-separated DDL statements; may be null or empty
 * @param statementConsumer the consumer that should be called with each sub-sequence of DDL statements that apply to
 *            a single database; may be null if no action is to be performed with the changes
 * @return {@code true} if changes were made to the database schema, or {@code false} if the DDL statements had no
 *         effect on the database schema
 */
public boolean applyDdl(SourceInfo source, String databaseName, String ddlStatements, DatabaseStatementStringConsumer statementConsumer) {
    Set<TableId> changes;
    if (ignoredQueryStatements.contains(ddlStatements))
        return false;
    try {
        this.ddlChanges.reset();
        this.ddlParser.setCurrentSchema(databaseName);
        this.ddlParser.parse(ddlStatements, tables);
    } catch (ParsingException e) {
        if (skipUnparseableDDL) {
            logger.warn("Ignoring unparseable DDL statement '{}': {}", ddlStatements);
        } else {
            throw e;
        }
    } finally {
        changes = tables.drainChanges();
        // for controlling this, too
        if (!storeOnlyMonitoredTablesDdl || !changes.isEmpty()) {
            if (statementConsumer != null) {
                if (!ddlChanges.isEmpty() && ddlChanges.applyToMoreDatabasesThan(databaseName)) {
                    // We understood at least some of the DDL statements and can figure out to which database they apply.
                    // They also apply to more databases than 'databaseName', so we need to apply the DDL statements in
                    // the same order they were read for each _affected_ database, grouped together if multiple apply
                    // to the same _affected_ database...
                    ddlChanges.groupStatementStringsByDatabase((dbName, ddl) -> {
                        if (filters.databaseFilter().test(dbName) || dbName == null || "".equals(dbName)) {
                            if (dbName == null)
                                dbName = "";
                            statementConsumer.consume(dbName, ddlStatements);
                        }
                    });
                } else if (filters.databaseFilter().test(databaseName) || databaseName == null || "".equals(databaseName)) {
                    if (databaseName == null)
                        databaseName = "";
                    statementConsumer.consume(databaseName, ddlStatements);
                }
            }
            // schema change records.
            try {
                if (!storeOnlyMonitoredTablesDdl || changes.stream().anyMatch(filters().tableFilter()::test)) {
                    dbHistory.record(source.partition(), source.offset(), databaseName, ddlStatements);
                } else {
                    logger.debug("Changes for DDL '{}' were filtered and not recorded in database history", ddlStatements);
                }
            } catch (Throwable e) {
                throw new ConnectException("Error recording the DDL statement(s) in the database history " + dbHistory + ": " + ddlStatements, e);
            }
        }
    }
    // Figure out what changed ...
    changes.forEach(tableId -> {
        Table table = tables.forTable(tableId);
        if (table == null) {
            // removed
            tableSchemaByTableId.remove(tableId);
        } else {
            TableSchema schema = schemaBuilder.create(schemaPrefix, getEnvelopeSchemaName(table), table, filters.columnFilter(), filters.columnMappers());
            tableSchemaByTableId.put(tableId, schema);
        }
    });
    return true;
}

Example 27 with Table

use of io.debezium.relational.Table in project debezium by debezium.

the class RecordsSnapshotProducer method readTable.

private void readTable(TableId tableId, ResultSet rs, BlockingConsumer<ChangeEvent> consumer, AtomicInteger rowsCounter) throws SQLException, InterruptedException {
    Table table = schema().tableFor(tableId);
    assert table != null;
    final int numColumns = table.columns().size();
    final Object[] row = new Object[numColumns];
    final ResultSetMetaData metaData = rs.getMetaData();
    while (rs.next()) {
        rowsCounter.incrementAndGet();
        sendCurrentRecord(consumer);
        for (int i = 0, j = 1; i != numColumns; ++i, ++j) {
            row[i] = valueForColumn(rs, j, metaData);
        }
        generateReadRecord(tableId, row);
    }
}

Example 28 with Table

use of io.debezium.relational.Table in project debezium by debezium.

the class MySqlDdlParser method parseCreateTable.

protected void parseCreateTable(Marker start) {
    tokens.canConsume("TEMPORARY");
    tokens.consume("TABLE");
    boolean onlyIfNotExists = tokens.canConsume("IF", "NOT", "EXISTS");
    TableId tableId = parseQualifiedTableName(start);
    if (tokens.canConsume("LIKE")) {
        TableId originalId = parseQualifiedTableName(start);
        Table original = databaseTables.forTable(originalId);
        if (original != null) {
            databaseTables.overwriteTable(tableId, original.columns(), original.primaryKeyColumnNames(), original.defaultCharsetName());
        }
        consumeRemainingStatement(start);
        signalCreateTable(tableId, start);
        debugParsed(start);
        return;
    }
    if (onlyIfNotExists && databaseTables.forTable(tableId) != null) {
        // The table does exist, so we should do nothing ...
        consumeRemainingStatement(start);
        signalCreateTable(tableId, start);
        debugParsed(start);
        return;
    }
    TableEditor table = databaseTables.editOrCreateTable(tableId);
    // create_definition ...
    if (tokens.matches('('))
        parseCreateDefinitionList(start, table);
    // table_options ...
    parseTableOptions(start, table);
    // partition_options ...
    if (tokens.matches("PARTITION")) {
        parsePartitionOptions(start, table);
    }
    // select_statement
    if (tokens.canConsume("AS") || tokens.canConsume("IGNORE", "AS") || tokens.canConsume("REPLACE", "AS")) {
        parseAsSelectStatement(start, table);
    }
    // Make sure that the table's character set has been set ...
    if (!table.hasDefaultCharsetName()) {
        table.setDefaultCharsetName(currentDatabaseCharset());
    }
    // Update the table definition ...
    databaseTables.overwriteTable(table.create());
    signalCreateTable(tableId, start);
    debugParsed(start);
}

Example 29 with Table

use of io.debezium.relational.Table in project debezium by debezium.

the class PostgresSchema method refreshSchema.

private void refreshSchema(TableId id) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("refreshing DB schema for table '{}'", id);
    }
    Table table = this.tables.forTable(id);
    TableSchema schema = schemaBuilder.create(schemaPrefix, getEnvelopeSchemaName(table), table, filters.columnFilter(), null);
    tableSchemaByTableId.put(id, schema);
}

Example 30 with Table

use of io.debezium.relational.Table in project debezium by debezium.

the class RecordsStreamProducer method columnValues.

private Object[] columnValues(List<ReplicationMessage.Column> columns, TableId tableId, boolean refreshSchemaIfChanged, boolean metadataInMessage) throws SQLException {
    if (columns == null || columns.isEmpty()) {
        return null;
    }
    Table table = schema().tableFor(tableId);
    assert table != null;
    // check if we need to refresh our local schema due to DB schema changes for this table
    if (refreshSchemaIfChanged && schemaChanged(columns, table, metadataInMessage)) {
        try (final PostgresConnection connection = taskContext.createConnection()) {
            // Refresh the schema so we get information about primary keys
            schema().refresh(connection, tableId);
            // Update the schema with metadata coming from decoder message
            if (metadataInMessage) {
                schema().refresh(tableFromFromMessage(columns, schema().tableFor(tableId)));
            }
            table = schema().tableFor(tableId);
        }
    }
    // based on the schema columns, create the values on the same position as the columns
    List<String> columnNames = table.columnNames();
    // JSON does not deliver a list of all columns for REPLICA IDENTITY DEFAULT
    Object[] values = new Object[columns.size() < columnNames.size() ? columnNames.size() : columns.size()];
    columns.forEach(message -> {
        // DBZ-298 Quoted column names will be sent like that in messages, but stored unquoted in the column names
        String columnName = Strings.unquoteIdentifierPart(message.getName());
        int position = columnNames.indexOf(columnName);
        assert position >= 0;
        values[position] = message.getValue(this::typeResolverConnection, taskContext.config().includeUnknownDatatypes());
    });
    return values;
}