Examples with Column io.debezium.relational.Column used on opensource projects

Example 6 with Column

use of io.debezium.relational.Column in project debezium by debezium.

the class MySqlValueConverters method converter.

@Override
public ValueConverter converter(Column column, Field fieldDefn) {
    // Handle a few MySQL-specific types based upon how they are handled by the MySQL binlog client ...
    String typeName = column.typeName().toUpperCase();
    if (matches(typeName, "JSON")) {
        return (data) -> convertJson(column, fieldDefn, data);
    }
    if (matches(typeName, "GEOMETRY") || matches(typeName, "LINESTRING") || matches(typeName, "POLYGON") || matches(typeName, "MULTIPOINT") || matches(typeName, "MULTILINESTRING") || matches(typeName, "MULTIPOLYGON") || matches(typeName, "GEOMETRYCOLLECTION")) {
        return (data -> convertGeometry(column, fieldDefn, data));
    }
    if (matches(typeName, "POINT")) {
        // backwards compatibility
        return (data -> convertPoint(column, fieldDefn, data));
    }
    if (matches(typeName, "YEAR")) {
        return (data) -> convertYearToInt(column, fieldDefn, data);
    }
    if (matches(typeName, "ENUM")) {
        // Build up the character array based upon the column's type ...
        List<String> options = extractEnumAndSetOptions(column);
        return (data) -> convertEnumToString(options, column, fieldDefn, data);
    }
    if (matches(typeName, "SET")) {
        // Build up the character array based upon the column's type ...
        List<String> options = extractEnumAndSetOptions(column);
        return (data) -> convertSetToString(options, column, fieldDefn, data);
    }
    if (matches(typeName, "TINYINT UNSIGNED") || matches(typeName, "TINYINT UNSIGNED ZEROFILL")) {
        // Convert TINYINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedTinyint(column, fieldDefn, data);
    }
    if (matches(typeName, "SMALLINT UNSIGNED") || matches(typeName, "SMALLINT UNSIGNED ZEROFILL")) {
        // Convert SMALLINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedSmallint(column, fieldDefn, data);
    }
    if (matches(typeName, "MEDIUMINT UNSIGNED") || matches(typeName, "MEDIUMINT UNSIGNED ZEROFILL")) {
        // Convert SMALLINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedMediumint(column, fieldDefn, data);
    }
    if (matches(typeName, "INT UNSIGNED") || matches(typeName, "INT UNSIGNED ZEROFILL")) {
        // Convert INT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedInt(column, fieldDefn, data);
    }
    if (matches(typeName, "BIGINT UNSIGNED") || matches(typeName, "BIGINT UNSIGNED ZEROFILL")) {
        switch(super.bigIntUnsignedMode) {
            case LONG:
                return (data) -> convertBigInt(column, fieldDefn, data);
            case PRECISE:
                // Convert BIGINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
                return (data) -> convertUnsignedBigint(column, fieldDefn, data);
        }
    }
    // We have to convert bytes encoded in the column's character set ...
    switch(column.jdbcType()) {
        // variable-length
        case Types.CHAR:
        // variable-length
        case Types.VARCHAR:
        // variable-length
        case Types.LONGVARCHAR:
        // variable-length
        case Types.CLOB:
        // fixed-length
        case Types.NCHAR:
        // fixed-length
        case Types.NVARCHAR:
        // fixed-length
        case Types.LONGNVARCHAR:
        // fixed-length
        case Types.NCLOB:
        case Types.DATALINK:
        case Types.SQLXML:
            Charset charset = charsetFor(column);
            if (charset != null) {
                logger.debug("Using {} charset by default for column: {}", charset, column);
                return (data) -> convertString(column, fieldDefn, charset, data);
            }
            logger.warn("Using UTF-8 charset by default for column without charset: {}", column);
            return (data) -> convertString(column, fieldDefn, StandardCharsets.UTF_8, data);
        case Types.TIME:
            if (adaptiveTimeMicrosecondsPrecisionMode)
                return data -> convertDurationToMicroseconds(column, fieldDefn, data);
        default:
            break;
    }
    // Otherwise, let the base class handle it ...
    return super.converter(column, fieldDefn);
}

Example 7 with Column

use of io.debezium.relational.Column in project debezium by debezium.

the class RecordsStreamProducer method schemaChanged.

private boolean schemaChanged(List<ReplicationMessage.Column> columns, Table table, boolean metadataInMessage) {
    List<String> columnNames = table.columnNames();
    int messagesCount = columns.size();
    if (columnNames.size() != messagesCount) {
        // so we need to trigger a refresh...
        return true;
    }
    // on what we have in the table metadata....
    return columns.stream().filter(message -> {
        String columnName = message.getName();
        Column column = table.columnWithName(columnName);
        if (column == null) {
            logger.info("found new column '{}' present in the server message which is not part of the table metadata; refreshing table schema", columnName);
            return true;
        } else {
            final int localType = column.nativeType();
            final int incomingType = message.getType().getOid();
            if (localType != incomingType) {
                logger.info("detected new type for column '{}', old type was {} ({}), new type is {} ({}); refreshing table schema", columnName, localType, column.typeName(), incomingType, message.getType().getName());
                return true;
            }
            if (metadataInMessage) {
                final int localLength = column.length();
                final int incomingLength = message.getTypeMetadata().getLength();
                if (localLength != incomingLength) {
                    logger.info("detected new length for column '{}', old length was {}, new length is {}; refreshing table schema", columnName, localLength, incomingLength);
                    return true;
                }
                final int localScale = column.scale();
                final int incomingScale = message.getTypeMetadata().getScale();
                if (localScale != incomingScale) {
                    logger.info("detected new scale for column '{}', old scale was {}, new scale is {}; refreshing table schema", columnName, localScale, incomingScale);
                    return true;
                }
            }
        }
        return false;
    }).findFirst().isPresent();
}

Example 8 with Column

use of io.debezium.relational.Column in project debezium by debezium.

the class MySqlDdlParser method parseCreateDefinition.

/**
 * @param isAlterStatement whether this is an ALTER TABLE statement or not (i.e. CREATE TABLE)
 */
protected void parseCreateDefinition(Marker start, TableEditor table, boolean isAlterStatement) {
    // If the first token is a quoted identifier, then we know it is a column name ...
    Collection<ParsingException> errors = null;
    boolean quoted = isNextTokenQuotedIdentifier();
    Marker defnStart = tokens.mark();
    if (!quoted) {
        // The first token is not quoted so let's check for other expressions ...
        if (tokens.canConsume("CHECK")) {
            // Try to parse the constraints first ...
            consumeExpression(start);
            return;
        }
        if (tokens.canConsume("CONSTRAINT", TokenStream.ANY_VALUE, "PRIMARY", "KEY") || tokens.canConsume("CONSTRAINT", "PRIMARY", "KEY") || tokens.canConsume("PRIMARY", "KEY")) {
            try {
                if (tokens.canConsume("USING")) {
                    parseIndexType(start);
                }
                if (!tokens.matches('(')) {
                    // index name
                    tokens.consume();
                }
                List<String> pkColumnNames = parseIndexColumnNames(start);
                table.setPrimaryKeyNames(pkColumnNames);
                parseIndexOptions(start);
                // MySQL does not allow a primary key to have nullable columns, so let's make sure we model that correctly ...
                pkColumnNames.forEach(name -> {
                    Column c = table.columnWithName(name);
                    if (c != null && c.isOptional()) {
                        table.addColumn(c.edit().optional(false).create());
                    }
                });
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsume("CONSTRAINT", TokenStream.ANY_VALUE, "UNIQUE") || tokens.canConsume("CONSTRAINT", "UNIQUE") || tokens.canConsume("UNIQUE")) {
            tokens.canConsumeAnyOf("KEY", "INDEX");
            try {
                if (!tokens.matches('(')) {
                    if (!tokens.matches("USING")) {
                        // name of unique index ...
                        tokens.consume();
                    }
                    if (tokens.matches("USING")) {
                        parseIndexType(start);
                    }
                }
                List<String> uniqueKeyColumnNames = parseIndexColumnNames(start);
                if (table.primaryKeyColumnNames().isEmpty()) {
                    // this may eventually get overwritten by a real PK
                    table.setPrimaryKeyNames(uniqueKeyColumnNames);
                }
                parseIndexOptions(start);
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsume("CONSTRAINT", TokenStream.ANY_VALUE, "FOREIGN", "KEY") || tokens.canConsume("FOREIGN", "KEY")) {
            try {
                if (!tokens.matches('(')) {
                    // name of foreign key
                    tokens.consume();
                }
                parseIndexColumnNames(start);
                if (tokens.matches("REFERENCES")) {
                    parseReferenceDefinition(start);
                }
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsumeAnyOf("INDEX", "KEY")) {
            try {
                if (!tokens.matches('(')) {
                    if (!tokens.matches("USING")) {
                        // name of unique index ...
                        tokens.consume();
                    }
                    if (tokens.matches("USING")) {
                        parseIndexType(start);
                    }
                }
                parseIndexColumnNames(start);
                parseIndexOptions(start);
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
        if (tokens.canConsumeAnyOf("FULLTEXT", "SPATIAL")) {
            try {
                tokens.canConsumeAnyOf("INDEX", "KEY");
                if (!tokens.matches('(')) {
                    // name of unique index ...
                    tokens.consume();
                }
                parseIndexColumnNames(start);
                parseIndexOptions(start);
                return;
            } catch (ParsingException e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            } catch (MultipleParsingExceptions e) {
                // Invalid names, so rewind and continue
                errors = accumulateParsingFailure(e, errors);
                tokens.rewind(defnStart);
            }
        }
    }
    try {
        // It's either quoted (meaning it's a column definition)
        if (isAlterStatement && !quoted) {
            // optional for ALTER TABLE
            tokens.canConsume("COLUMN");
        }
        String columnName = parseColumnName();
        parseCreateColumn(start, table, columnName, null);
    } catch (ParsingException e) {
        if (errors != null) {
            errors = accumulateParsingFailure(e, errors);
            throw new MultipleParsingExceptions(errors);
        }
        throw e;
    } catch (MultipleParsingExceptions e) {
        if (errors != null) {
            errors = accumulateParsingFailure(e, errors);
            throw new MultipleParsingExceptions(errors);
        }
        throw e;
    }
}

Example 9 with Column

use of io.debezium.relational.Column in project debezium by debezium.

the class PostgresValueConverter method createArrayConverter.

private ValueConverter createArrayConverter(Column column, Field fieldDefn) {
    PostgresType arrayType = typeRegistry.get(column.nativeType());
    PostgresType elementType = arrayType.getElementType();
    final String elementTypeName = elementType.getName();
    final String elementColumnName = column.name() + "-element";
    final Column elementColumn = Column.editor().name(elementColumnName).jdbcType(elementType.getJdbcId()).nativeType(elementType.getOid()).type(elementTypeName).optional(true).scale(column.scale()).length(column.length()).create();
    Schema elementSchema = schemaBuilder(elementColumn).optional().build();
    final Field elementField = new Field(elementColumnName, 0, elementSchema);
    final ValueConverter elementConverter = converter(elementColumn, elementField);
    return data -> convertArray(column, fieldDefn, elementConverter, data);
}

Example 10 with Column

use of io.debezium.relational.Column in project debezium by debezium.

the class MySqlDdlParserTest method assertColumn.

protected void assertColumn(Table table, String name, String typeName, int jdbcType, int length, String charsetName, boolean optional) {
    Column column = table.columnWithName(name);
    assertThat(column.name()).isEqualTo(name);
    assertThat(column.typeName()).isEqualTo(typeName);
    assertThat(column.jdbcType()).isEqualTo(jdbcType);
    assertThat(column.length()).isEqualTo(length);
    assertThat(column.charsetName()).isEqualTo(charsetName);
    assertThat(column.scale()).isEqualTo(-1);
    assertThat(column.isOptional()).isEqualTo(optional);
    assertThat(column.isGenerated()).isFalse();
    assertThat(column.isAutoIncremented()).isFalse();
}