Examples with TableSchema io.debezium.relational.TableSchema used on opensource projects

Example 6 with TableSchema

use of io.debezium.relational.TableSchema in project debezium by debezium.

the class MySqlSchema method refreshSchemas.

/**
 * Discard any currently-cached schemas and rebuild them using the filters.
 */
protected void refreshSchemas() {
    tableSchemaByTableId.clear();
    // Create TableSchema instances for any existing table ...
    this.tables.tableIds().forEach(id -> {
        Table table = this.tables.forTable(id);
        TableSchema schema = schemaBuilder.create(schemaPrefix, getEnvelopeSchemaName(table), table, filters.columnFilter(), filters.columnMappers());
        tableSchemaByTableId.put(id, schema);
    });
}

Example 7 with TableSchema

use of io.debezium.relational.TableSchema in project debezium by debezium.

the class MySqlSchema method applyDdl.

/**
 * Apply the supplied DDL statements to this database schema and record the history. If a {@code statementConsumer} is
 * supplied, then call it for each sub-sequence of the DDL statements that all apply to the same database.
 * <p>
 * Typically DDL statements are applied using a connection to a single database, and unless the statements use fully-qualified
 * names, the DDL statements apply to this database.
 *
 * @param source the current {@link SourceInfo#partition()} and {@link SourceInfo#offset() offset} at which these changes are
 *            found; may not be null
 * @param databaseName the name of the default database under which these statements are applied; may not be null
 * @param ddlStatements the {@code ;}-separated DDL statements; may be null or empty
 * @param statementConsumer the consumer that should be called with each sub-sequence of DDL statements that apply to
 *            a single database; may be null if no action is to be performed with the changes
 * @return {@code true} if changes were made to the database schema, or {@code false} if the DDL statements had no
 *         effect on the database schema
 */
public boolean applyDdl(SourceInfo source, String databaseName, String ddlStatements, DatabaseStatementStringConsumer statementConsumer) {
    Set<TableId> changes;
    if (ignoredQueryStatements.contains(ddlStatements))
        return false;
    try {
        this.ddlChanges.reset();
        this.ddlParser.setCurrentSchema(databaseName);
        this.ddlParser.parse(ddlStatements, tables);
    } catch (ParsingException e) {
        if (skipUnparseableDDL) {
            logger.warn("Ignoring unparseable DDL statement '{}': {}", ddlStatements);
        } else {
            throw e;
        }
    } finally {
        changes = tables.drainChanges();
        // for controlling this, too
        if (!storeOnlyMonitoredTablesDdl || !changes.isEmpty()) {
            if (statementConsumer != null) {
                if (!ddlChanges.isEmpty() && ddlChanges.applyToMoreDatabasesThan(databaseName)) {
                    // We understood at least some of the DDL statements and can figure out to which database they apply.
                    // They also apply to more databases than 'databaseName', so we need to apply the DDL statements in
                    // the same order they were read for each _affected_ database, grouped together if multiple apply
                    // to the same _affected_ database...
                    ddlChanges.groupStatementStringsByDatabase((dbName, ddl) -> {
                        if (filters.databaseFilter().test(dbName) || dbName == null || "".equals(dbName)) {
                            if (dbName == null)
                                dbName = "";
                            statementConsumer.consume(dbName, ddlStatements);
                        }
                    });
                } else if (filters.databaseFilter().test(databaseName) || databaseName == null || "".equals(databaseName)) {
                    if (databaseName == null)
                        databaseName = "";
                    statementConsumer.consume(databaseName, ddlStatements);
                }
            }
            // schema change records.
            try {
                if (!storeOnlyMonitoredTablesDdl || changes.stream().anyMatch(filters().tableFilter()::test)) {
                    dbHistory.record(source.partition(), source.offset(), databaseName, ddlStatements);
                } else {
                    logger.debug("Changes for DDL '{}' were filtered and not recorded in database history", ddlStatements);
                }
            } catch (Throwable e) {
                throw new ConnectException("Error recording the DDL statement(s) in the database history " + dbHistory + ": " + ddlStatements, e);
            }
        }
    }
    // Figure out what changed ...
    changes.forEach(tableId -> {
        Table table = tables.forTable(tableId);
        if (table == null) {
            // removed
            tableSchemaByTableId.remove(tableId);
        } else {
            TableSchema schema = schemaBuilder.create(schemaPrefix, getEnvelopeSchemaName(table), table, filters.columnFilter(), filters.columnMappers());
            tableSchemaByTableId.put(tableId, schema);
        }
    });
    return true;
}

Example 8 with TableSchema

use of io.debezium.relational.TableSchema in project debezium by debezium.

the class RecordsStreamProducer method generateCreateRecord.

protected void generateCreateRecord(TableId tableId, Object[] rowData, boolean isLastEventForLsn, BlockingConsumer<ChangeEvent> recordConsumer) throws InterruptedException {
    if (rowData == null || rowData.length == 0) {
        logger.warn("no new values found for table '{}' from update message at '{}';skipping record", tableId, sourceInfo);
        return;
    }
    TableSchema tableSchema = schema().schemaFor(tableId);
    assert tableSchema != null;
    Object key = tableSchema.keyFromColumnData(rowData);
    Struct value = tableSchema.valueFromColumnData(rowData);
    if (key == null || value == null) {
        return;
    }
    Schema keySchema = tableSchema.keySchema();
    Map<String, ?> partition = sourceInfo.partition();
    Map<String, ?> offset = sourceInfo.offset();
    String topicName = topicSelector().topicNameFor(tableId);
    Envelope envelope = tableSchema.getEnvelopeSchema();
    SourceRecord record = new SourceRecord(partition, offset, topicName, null, keySchema, key, envelope.schema(), envelope.create(value, sourceInfo.source(), clock().currentTimeInMillis()));
    if (logger.isDebugEnabled()) {
        logger.debug("sending create event '{}' to topic '{}'", record, topicName);
    }
    recordConsumer.accept(new ChangeEvent(record, isLastEventForLsn));
}

Example 9 with TableSchema

use of io.debezium.relational.TableSchema in project debezium by debezium.

the class RecordsStreamProducer method process.

private void process(ReplicationMessage message, Long lsn, BlockingConsumer<ChangeEvent> consumer) throws SQLException, InterruptedException {
    if (message == null) {
        // in some cases we can get null if PG gives us back a message earlier than the latest reported flushed LSN
        return;
    }
    TableId tableId = PostgresSchema.parse(message.getTable());
    assert tableId != null;
    // update the source info with the coordinates for this message
    long commitTimeNs = message.getCommitTime();
    long txId = message.getTransactionId();
    sourceInfo.update(lsn, commitTimeNs, txId);
    if (logger.isDebugEnabled()) {
        logger.debug("received new message at position {}\n{}", ReplicationConnection.format(lsn), message);
    }
    TableSchema tableSchema = tableSchemaFor(tableId);
    if (tableSchema == null) {
        return;
    }
    if (tableSchema.keySchema() == null) {
        logger.warn("ignoring message for table '{}' because it does not have a primary key defined", tableId);
    }
    ReplicationMessage.Operation operation = message.getOperation();
    switch(operation) {
        case INSERT:
            {
                Object[] row = columnValues(message.getNewTupleList(), tableId, true, message.hasTypeMetadata());
                generateCreateRecord(tableId, row, message.isLastEventForLsn(), consumer);
                break;
            }
        case UPDATE:
            {
                Object[] newRow = columnValues(message.getNewTupleList(), tableId, true, message.hasTypeMetadata());
                Object[] oldRow = columnValues(message.getOldTupleList(), tableId, false, message.hasTypeMetadata());
                generateUpdateRecord(tableId, oldRow, newRow, message.isLastEventForLsn(), consumer);
                break;
            }
        case DELETE:
            {
                Object[] row = columnValues(message.getOldTupleList(), tableId, false, message.hasTypeMetadata());
                generateDeleteRecord(tableId, row, message.isLastEventForLsn(), consumer);
                break;
            }
        default:
            {
                logger.warn("unknown message operation: " + operation);
            }
    }
}

Example 10 with TableSchema

use of io.debezium.relational.TableSchema in project debezium by debezium.

the class PostgresSchema method refreshSchema.

private void refreshSchema(TableId id) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("refreshing DB schema for table '{}'", id);
    }
    Table table = this.tables.forTable(id);
    TableSchema schema = schemaBuilder.create(schemaPrefix, getEnvelopeSchemaName(table), table, filters.columnFilter(), null);
    tableSchemaByTableId.put(id, schema);
}