Search in sources :

Example 1 with StartupOptions

use of org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.StartupOptions in project flink by apache.

the class KafkaDynamicTableFactory method createDynamicTableSource.

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
    final TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
    final Optional<DecodingFormat<DeserializationSchema<RowData>>> keyDecodingFormat = getKeyDecodingFormat(helper);
    final DecodingFormat<DeserializationSchema<RowData>> valueDecodingFormat = getValueDecodingFormat(helper);
    helper.validateExcept(PROPERTIES_PREFIX);
    final ReadableConfig tableOptions = helper.getOptions();
    validateTableSourceOptions(tableOptions);
    validatePKConstraints(context.getObjectIdentifier(), context.getPrimaryKeyIndexes(), context.getCatalogTable().getOptions(), valueDecodingFormat);
    final StartupOptions startupOptions = getStartupOptions(tableOptions);
    final Properties properties = getKafkaProperties(context.getCatalogTable().getOptions());
    // add topic-partition discovery
    final Optional<Long> partitionDiscoveryInterval = tableOptions.getOptional(SCAN_TOPIC_PARTITION_DISCOVERY).map(Duration::toMillis);
    properties.setProperty(KafkaSourceOptions.PARTITION_DISCOVERY_INTERVAL_MS.key(), partitionDiscoveryInterval.orElse(-1L).toString());
    final DataType physicalDataType = context.getPhysicalRowDataType();
    final int[] keyProjection = createKeyFormatProjection(tableOptions, physicalDataType);
    final int[] valueProjection = createValueFormatProjection(tableOptions, physicalDataType);
    final String keyPrefix = tableOptions.getOptional(KEY_FIELDS_PREFIX).orElse(null);
    return createKafkaTableSource(physicalDataType, keyDecodingFormat.orElse(null), valueDecodingFormat, keyProjection, valueProjection, keyPrefix, getSourceTopics(tableOptions), getSourceTopicPattern(tableOptions), properties, startupOptions.startupMode, startupOptions.specificOffsets, startupOptions.startupTimestampMillis, context.getObjectIdentifier().asSummaryString());
}
Also used : DecodingFormat(org.apache.flink.table.connector.format.DecodingFormat) TableFactoryHelper(org.apache.flink.table.factories.FactoryUtil.TableFactoryHelper) Duration(java.time.Duration) KafkaConnectorOptionsUtil.getKafkaProperties(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.getKafkaProperties) Properties(java.util.Properties) DeserializationSchema(org.apache.flink.api.common.serialization.DeserializationSchema) RowData(org.apache.flink.table.data.RowData) ReadableConfig(org.apache.flink.configuration.ReadableConfig) DataType(org.apache.flink.table.types.DataType) KafkaConnectorOptionsUtil.getStartupOptions(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.getStartupOptions) StartupOptions(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.StartupOptions)

Aggregations

Duration (java.time.Duration)1 Properties (java.util.Properties)1 DeserializationSchema (org.apache.flink.api.common.serialization.DeserializationSchema)1 ReadableConfig (org.apache.flink.configuration.ReadableConfig)1 StartupOptions (org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.StartupOptions)1 KafkaConnectorOptionsUtil.getKafkaProperties (org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.getKafkaProperties)1 KafkaConnectorOptionsUtil.getStartupOptions (org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptionsUtil.getStartupOptions)1 DecodingFormat (org.apache.flink.table.connector.format.DecodingFormat)1 RowData (org.apache.flink.table.data.RowData)1 TableFactoryHelper (org.apache.flink.table.factories.FactoryUtil.TableFactoryHelper)1 DataType (org.apache.flink.table.types.DataType)1