Search in sources :

Example 1 with ValueConverter

use of io.debezium.relational.ValueConverter in project debezium by debezium.

the class MySqlValueConverters method converter.

@Override
public ValueConverter converter(Column column, Field fieldDefn) {
    // Handle a few MySQL-specific types based upon how they are handled by the MySQL binlog client ...
    String typeName = column.typeName().toUpperCase();
    if (matches(typeName, "JSON")) {
        return (data) -> convertJson(column, fieldDefn, data);
    }
    if (matches(typeName, "GEOMETRY") || matches(typeName, "LINESTRING") || matches(typeName, "POLYGON") || matches(typeName, "MULTIPOINT") || matches(typeName, "MULTILINESTRING") || matches(typeName, "MULTIPOLYGON") || matches(typeName, "GEOMETRYCOLLECTION")) {
        return (data -> convertGeometry(column, fieldDefn, data));
    }
    if (matches(typeName, "POINT")) {
        // backwards compatibility
        return (data -> convertPoint(column, fieldDefn, data));
    }
    if (matches(typeName, "YEAR")) {
        return (data) -> convertYearToInt(column, fieldDefn, data);
    }
    if (matches(typeName, "ENUM")) {
        // Build up the character array based upon the column's type ...
        List<String> options = extractEnumAndSetOptions(column);
        return (data) -> convertEnumToString(options, column, fieldDefn, data);
    }
    if (matches(typeName, "SET")) {
        // Build up the character array based upon the column's type ...
        List<String> options = extractEnumAndSetOptions(column);
        return (data) -> convertSetToString(options, column, fieldDefn, data);
    }
    if (matches(typeName, "TINYINT UNSIGNED") || matches(typeName, "TINYINT UNSIGNED ZEROFILL")) {
        // Convert TINYINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedTinyint(column, fieldDefn, data);
    }
    if (matches(typeName, "SMALLINT UNSIGNED") || matches(typeName, "SMALLINT UNSIGNED ZEROFILL")) {
        // Convert SMALLINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedSmallint(column, fieldDefn, data);
    }
    if (matches(typeName, "MEDIUMINT UNSIGNED") || matches(typeName, "MEDIUMINT UNSIGNED ZEROFILL")) {
        // Convert SMALLINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedMediumint(column, fieldDefn, data);
    }
    if (matches(typeName, "INT UNSIGNED") || matches(typeName, "INT UNSIGNED ZEROFILL")) {
        // Convert INT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
        return (data) -> convertUnsignedInt(column, fieldDefn, data);
    }
    if (matches(typeName, "BIGINT UNSIGNED") || matches(typeName, "BIGINT UNSIGNED ZEROFILL")) {
        switch(super.bigIntUnsignedMode) {
            case LONG:
                return (data) -> convertBigInt(column, fieldDefn, data);
            case PRECISE:
                // Convert BIGINT UNSIGNED internally from SIGNED to UNSIGNED based on the boundary settings
                return (data) -> convertUnsignedBigint(column, fieldDefn, data);
        }
    }
    // We have to convert bytes encoded in the column's character set ...
    switch(column.jdbcType()) {
        // variable-length
        case Types.CHAR:
        // variable-length
        case Types.VARCHAR:
        // variable-length
        case Types.LONGVARCHAR:
        // variable-length
        case Types.CLOB:
        // fixed-length
        case Types.NCHAR:
        // fixed-length
        case Types.NVARCHAR:
        // fixed-length
        case Types.LONGNVARCHAR:
        // fixed-length
        case Types.NCLOB:
        case Types.DATALINK:
        case Types.SQLXML:
            Charset charset = charsetFor(column);
            if (charset != null) {
                logger.debug("Using {} charset by default for column: {}", charset, column);
                return (data) -> convertString(column, fieldDefn, charset, data);
            }
            logger.warn("Using UTF-8 charset by default for column without charset: {}", column);
            return (data) -> convertString(column, fieldDefn, StandardCharsets.UTF_8, data);
        case Types.TIME:
            if (adaptiveTimeMicrosecondsPrecisionMode)
                return data -> convertDurationToMicroseconds(column, fieldDefn, data);
        default:
            break;
    }
    // Otherwise, let the base class handle it ...
    return super.converter(column, fieldDefn);
}
Also used : ChronoField(java.time.temporal.ChronoField) Arrays(java.util.Arrays) Json(io.debezium.data.Json) ByteBuffer(java.nio.ByteBuffer) Schema(org.apache.kafka.connect.data.Schema) BigDecimal(java.math.BigDecimal) Year(io.debezium.time.Year) CharsetMapping(com.mysql.jdbc.CharsetMapping) Charset(java.nio.charset.Charset) Duration(java.time.Duration) IllegalCharsetNameException(java.nio.charset.IllegalCharsetNameException) ZoneOffset(java.time.ZoneOffset) JsonBinary(com.github.shyiko.mysql.binlog.event.deserialization.json.JsonBinary) Strings(io.debezium.util.Strings) AbstractRowsEventDataDeserializer(com.github.shyiko.mysql.binlog.event.deserialization.AbstractRowsEventDataDeserializer) Field(org.apache.kafka.connect.data.Field) TemporalPrecisionMode(io.debezium.jdbc.TemporalPrecisionMode) IOException(java.io.IOException) SourceRecord(org.apache.kafka.connect.source.SourceRecord) StandardCharsets(java.nio.charset.StandardCharsets) ByteOrder(java.nio.ByteOrder) Decimal(org.apache.kafka.connect.data.Decimal) List(java.util.List) OffsetDateTime(java.time.OffsetDateTime) ChronoUnit(java.time.temporal.ChronoUnit) Immutable(io.debezium.annotation.Immutable) Column(io.debezium.relational.Column) ConnectException(org.apache.kafka.connect.errors.ConnectException) JdbcValueConverters(io.debezium.jdbc.JdbcValueConverters) ValueConverter(io.debezium.relational.ValueConverter) SchemaBuilder(org.apache.kafka.connect.data.SchemaBuilder) Temporal(java.time.temporal.Temporal) Types(java.sql.Types) Charset(java.nio.charset.Charset)

Example 2 with ValueConverter

use of io.debezium.relational.ValueConverter in project debezium by debezium.

the class PostgresValueConverter method createArrayConverter.

private ValueConverter createArrayConverter(Column column, Field fieldDefn) {
    PostgresType arrayType = typeRegistry.get(column.nativeType());
    PostgresType elementType = arrayType.getElementType();
    final String elementTypeName = elementType.getName();
    final String elementColumnName = column.name() + "-element";
    final Column elementColumn = Column.editor().name(elementColumnName).jdbcType(elementType.getJdbcId()).nativeType(elementType.getOid()).type(elementTypeName).optional(true).scale(column.scale()).length(column.length()).create();
    Schema elementSchema = schemaBuilder(elementColumn).optional().build();
    final Field elementField = new Field(elementColumnName, 0, elementSchema);
    final ValueConverter elementConverter = converter(elementColumn, elementField);
    return data -> convertArray(column, fieldDefn, elementConverter, data);
}
Also used : PGInterval(org.postgresql.util.PGInterval) Json(io.debezium.data.Json) NumberConversions(io.debezium.util.NumberConversions) MicroDuration(io.debezium.time.MicroDuration) Date(java.util.Date) LocalDateTime(java.time.LocalDateTime) Schema(org.apache.kafka.connect.data.Schema) PGobject(org.postgresql.util.PGobject) BigDecimal(java.math.BigDecimal) SQLException(java.sql.SQLException) PgProto(io.debezium.connector.postgresql.proto.PgProto) ZonedTime(io.debezium.time.ZonedTime) LocalTime(java.time.LocalTime) VariableScaleDecimal(io.debezium.data.VariableScaleDecimal) BigInteger(java.math.BigInteger) ZoneOffset(java.time.ZoneOffset) Point(io.debezium.data.geometry.Point) Geography(io.debezium.data.geometry.Geography) ZonedTimestamp(io.debezium.time.ZonedTimestamp) OffsetTime(java.time.OffsetTime) Bits(io.debezium.data.Bits) Field(org.apache.kafka.connect.data.Field) TemporalPrecisionMode(io.debezium.jdbc.TemporalPrecisionMode) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) Decimal(org.apache.kafka.connect.data.Decimal) Geometry(io.debezium.data.geometry.Geometry) List(java.util.List) OffsetDateTime(java.time.OffsetDateTime) SpecialValueDecimal(io.debezium.data.SpecialValueDecimal) Column(io.debezium.relational.Column) JdbcValueConverters(io.debezium.jdbc.JdbcValueConverters) ValueConverter(io.debezium.relational.ValueConverter) Optional(java.util.Optional) SchemaBuilder(org.apache.kafka.connect.data.SchemaBuilder) Uuid(io.debezium.data.Uuid) UnsupportedEncodingException(java.io.UnsupportedEncodingException) Collections(java.util.Collections) PGpoint(org.postgresql.geometric.PGpoint) Field(org.apache.kafka.connect.data.Field) Column(io.debezium.relational.Column) Schema(org.apache.kafka.connect.data.Schema) ValueConverter(io.debezium.relational.ValueConverter)

Aggregations

Json (io.debezium.data.Json)2 JdbcValueConverters (io.debezium.jdbc.JdbcValueConverters)2 TemporalPrecisionMode (io.debezium.jdbc.TemporalPrecisionMode)2 Column (io.debezium.relational.Column)2 ValueConverter (io.debezium.relational.ValueConverter)2 BigDecimal (java.math.BigDecimal)2 OffsetDateTime (java.time.OffsetDateTime)2 ZoneOffset (java.time.ZoneOffset)2 List (java.util.List)2 Decimal (org.apache.kafka.connect.data.Decimal)2 Field (org.apache.kafka.connect.data.Field)2 Schema (org.apache.kafka.connect.data.Schema)2 SchemaBuilder (org.apache.kafka.connect.data.SchemaBuilder)2 AbstractRowsEventDataDeserializer (com.github.shyiko.mysql.binlog.event.deserialization.AbstractRowsEventDataDeserializer)1 JsonBinary (com.github.shyiko.mysql.binlog.event.deserialization.json.JsonBinary)1 CharsetMapping (com.mysql.jdbc.CharsetMapping)1 Immutable (io.debezium.annotation.Immutable)1 PgProto (io.debezium.connector.postgresql.proto.PgProto)1 Bits (io.debezium.data.Bits)1 SpecialValueDecimal (io.debezium.data.SpecialValueDecimal)1