Examples with MetadataConverter com.ververica.cdc.debezium.table.MetadataConverter used on opensource projects

Example 1 with MetadataConverter

use of com.ververica.cdc.debezium.table.MetadataConverter in project flink-cdc-connectors by ververica.

the class MongoDBTableSource method getScanRuntimeProvider.

@Override
public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) {
    RowType physicalDataType = (RowType) physicalSchema.toPhysicalRowDataType().getLogicalType();
    MetadataConverter[] metadataConverters = getMetadataConverters();
    TypeInformation<RowData> typeInfo = scanContext.createTypeInformation(producedDataType);
    DebeziumDeserializationSchema<RowData> deserializer = new MongoDBConnectorDeserializationSchema(physicalDataType, metadataConverters, typeInfo, localTimeZone);
    MongoDBSource.Builder<RowData> builder = MongoDBSource.<RowData>builder().hosts(hosts).deserializer(deserializer);
    if (StringUtils.isNotEmpty(database) && StringUtils.isNotEmpty(collection)) {
        // explicitly specified database and collection.
        if (!containsRegexMetaCharacters(database) && !containsRegexMetaCharacters(collection)) {
            checkDatabaseNameValidity(database);
            checkCollectionNameValidity(collection);
            builder.databaseList(database);
            builder.collectionList(database + "." + collection);
        } else {
            builder.databaseList(database);
            builder.collectionList(collection);
        }
    } else if (StringUtils.isNotEmpty(database)) {
        builder.databaseList(database);
    } else if (StringUtils.isNotEmpty(collection)) {
        builder.collectionList(collection);
    } else {
    // Watching all changes on the cluster by default, we do nothing here
    }
    Optional.ofNullable(username).ifPresent(builder::username);
    Optional.ofNullable(password).ifPresent(builder::password);
    Optional.ofNullable(connectionOptions).ifPresent(builder::connectionOptions);
    Optional.ofNullable(errorsLogEnable).ifPresent(builder::errorsLogEnable);
    Optional.ofNullable(errorsTolerance).ifPresent(builder::errorsTolerance);
    Optional.ofNullable(copyExisting).ifPresent(builder::copyExisting);
    Optional.ofNullable(copyExistingPipeline).ifPresent(builder::copyExistingPipeline);
    Optional.ofNullable(copyExistingMaxThreads).ifPresent(builder::copyExistingMaxThreads);
    Optional.ofNullable(copyExistingQueueSize).ifPresent(builder::copyExistingQueueSize);
    Optional.ofNullable(pollMaxBatchSize).ifPresent(builder::pollMaxBatchSize);
    Optional.ofNullable(pollAwaitTimeMillis).ifPresent(builder::pollAwaitTimeMillis);
    Optional.ofNullable(heartbeatIntervalMillis).ifPresent(builder::heartbeatIntervalMillis);
    DebeziumSourceFunction<RowData> sourceFunction = builder.build();
    return SourceFunctionProvider.of(sourceFunction, false);
}

Example 2 with MetadataConverter

use of com.ververica.cdc.debezium.table.MetadataConverter in project flink-cdc-connectors by ververica.

the class OceanBaseTableSource method getScanRuntimeProvider.

@Override
public ScanRuntimeProvider getScanRuntimeProvider(ScanContext context) {
    RowType physicalDataType = (RowType) physicalSchema.toPhysicalRowDataType().getLogicalType();
    MetadataConverter[] metadataConverters = getMetadataConverters();
    TypeInformation<RowData> resultTypeInfo = context.createTypeInformation(producedDataType);
    DebeziumDeserializationSchema<RowData> deserializer = RowDataDebeziumDeserializeSchema.newBuilder().setPhysicalRowType(physicalDataType).setMetadataConverters(metadataConverters).setResultTypeInfo(resultTypeInfo).setServerTimeZone(serverTimeZone).build();
    OceanBaseSource.Builder<RowData> builder = OceanBaseSource.<RowData>builder().startupMode(startupMode).startupTimestamp(startupTimestamp).username(username).password(password).tenantName(tenantName).databaseName(databaseName).tableName(tableName).hostname(hostname).port(port).connectTimeout(connectTimeout).rsList(rsList).logProxyHost(logProxyHost).logProxyPort(logProxyPort).serverTimeZone(serverTimeZone).deserializer(deserializer);
    return SourceFunctionProvider.of(builder.build(), false);
}

Example 3 with MetadataConverter

use of com.ververica.cdc.debezium.table.MetadataConverter in project flink-cdc-connectors by ververica.

the class SqlServerTableSource method getScanRuntimeProvider.

@Override
public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) {
    RowType physicalDataType = (RowType) physicalSchema.toPhysicalRowDataType().getLogicalType();
    MetadataConverter[] metadataConverters = getMetadataConverters();
    TypeInformation<RowData> typeInfo = scanContext.createTypeInformation(producedDataType);
    DebeziumDeserializationSchema<RowData> deserializer = RowDataDebeziumDeserializeSchema.newBuilder().setPhysicalRowType(physicalDataType).setMetadataConverters(metadataConverters).setResultTypeInfo(typeInfo).setServerTimeZone(serverTimeZone).setUserDefinedConverterFactory(SqlServerDeserializationConverterFactory.instance()).build();
    DebeziumSourceFunction<RowData> sourceFunction = SqlServerSource.<RowData>builder().hostname(hostname).port(port).database(database).tableList(schemaName + "." + tableName).username(username).password(password).debeziumProperties(dbzProperties).startupOptions(startupOptions).deserializer(deserializer).build();
    return SourceFunctionProvider.of(sourceFunction, false);
}

Example 4 with MetadataConverter

use of com.ververica.cdc.debezium.table.MetadataConverter in project flink-cdc-connectors by ververica.

the class OracleTableSource method getScanRuntimeProvider.

@Override
public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) {
    RowType physicalDataType = (RowType) physicalSchema.toPhysicalRowDataType().getLogicalType();
    MetadataConverter[] metadataConverters = getMetadataConverters();
    TypeInformation<RowData> typeInfo = scanContext.createTypeInformation(producedDataType);
    DebeziumDeserializationSchema<RowData> deserializer = RowDataDebeziumDeserializeSchema.newBuilder().setPhysicalRowType(physicalDataType).setMetadataConverters(metadataConverters).setResultTypeInfo(typeInfo).setUserDefinedConverterFactory(OracleDeserializationConverterFactory.instance()).build();
    OracleSource.Builder<RowData> builder = OracleSource.<RowData>builder().hostname(hostname).port(port).database(database).tableList(schemaName + "." + tableName).schemaList(schemaName).username(username).password(password).debeziumProperties(dbzProperties).startupOptions(startupOptions).deserializer(deserializer);
    DebeziumSourceFunction<RowData> sourceFunction = builder.build();
    return SourceFunctionProvider.of(sourceFunction, false);
}

Example 5 with MetadataConverter

use of com.ververica.cdc.debezium.table.MetadataConverter in project flink-cdc-connectors by ververica.

the class MySqlTableSource method getScanRuntimeProvider.

@Override
public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) {
    RowType physicalDataType = (RowType) physicalSchema.toPhysicalRowDataType().getLogicalType();
    MetadataConverter[] metadataConverters = getMetadataConverters();
    final TypeInformation<RowData> typeInfo = scanContext.createTypeInformation(producedDataType);
    DebeziumDeserializationSchema<RowData> deserializer = RowDataDebeziumDeserializeSchema.newBuilder().setPhysicalRowType(physicalDataType).setMetadataConverters(metadataConverters).setResultTypeInfo(typeInfo).setServerTimeZone(serverTimeZone).setUserDefinedConverterFactory(MySqlDeserializationConverterFactory.instance()).build();
    if (enableParallelRead) {
        MySqlSource<RowData> parallelSource = MySqlSource.<RowData>builder().hostname(hostname).port(port).databaseList(database).tableList(database + "." + tableName).username(username).password(password).serverTimeZone(serverTimeZone.toString()).serverId(serverId).splitSize(splitSize).splitMetaGroupSize(splitMetaGroupSize).distributionFactorUpper(distributionFactorUpper).distributionFactorLower(distributionFactorLower).fetchSize(fetchSize).connectTimeout(connectTimeout).connectMaxRetries(connectMaxRetries).connectionPoolSize(connectionPoolSize).debeziumProperties(dbzProperties).startupOptions(startupOptions).deserializer(deserializer).scanNewlyAddedTableEnabled(scanNewlyAddedTableEnabled).jdbcProperties(jdbcProperties).heartbeatInterval(heartbeatInterval).build();
        return SourceProvider.of(parallelSource);
    } else {
        com.ververica.cdc.connectors.mysql.MySqlSource.Builder<RowData> builder = com.ververica.cdc.connectors.mysql.MySqlSource.<RowData>builder().hostname(hostname).port(port).databaseList(database).tableList(database + "." + tableName).username(username).password(password).serverTimeZone(serverTimeZone.toString()).debeziumProperties(dbzProperties).startupOptions(startupOptions).deserializer(deserializer);
        Optional.ofNullable(serverId).ifPresent(serverId -> builder.serverId(Integer.parseInt(serverId)));
        DebeziumSourceFunction<RowData> sourceFunction = builder.build();
        return SourceFunctionProvider.of(sourceFunction, false);
    }
}