Search in sources :

Example 1 with VALUE_FIELDS_INCLUDE

use of org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.VALUE_FIELDS_INCLUDE in project flink by apache.

the class KafkaConnectorOptionsUtil method createValueFormatProjection.

/**
 * Creates an array of indices that determine which physical fields of the table schema to
 * include in the value format.
 *
 * <p>See {@link KafkaConnectorOptions#VALUE_FORMAT}, {@link
 * KafkaConnectorOptions#VALUE_FIELDS_INCLUDE}, and {@link
 * KafkaConnectorOptions#KEY_FIELDS_PREFIX} for more information.
 */
public static int[] createValueFormatProjection(ReadableConfig options, DataType physicalDataType) {
    final LogicalType physicalType = physicalDataType.getLogicalType();
    Preconditions.checkArgument(physicalType.is(LogicalTypeRoot.ROW), "Row data type expected.");
    final int physicalFieldCount = LogicalTypeChecks.getFieldCount(physicalType);
    final IntStream physicalFields = IntStream.range(0, physicalFieldCount);
    final String keyPrefix = options.getOptional(KEY_FIELDS_PREFIX).orElse("");
    final ValueFieldsStrategy strategy = options.get(VALUE_FIELDS_INCLUDE);
    if (strategy == ValueFieldsStrategy.ALL) {
        if (keyPrefix.length() > 0) {
            throw new ValidationException(String.format("A key prefix is not allowed when option '%s' is set to '%s'. " + "Set it to '%s' instead to avoid field overlaps.", VALUE_FIELDS_INCLUDE.key(), ValueFieldsStrategy.ALL, ValueFieldsStrategy.EXCEPT_KEY));
        }
        return physicalFields.toArray();
    } else if (strategy == ValueFieldsStrategy.EXCEPT_KEY) {
        final int[] keyProjection = createKeyFormatProjection(options, physicalDataType);
        return physicalFields.filter(pos -> IntStream.of(keyProjection).noneMatch(k -> k == pos)).toArray();
    }
    throw new TableException("Unknown value fields strategy:" + strategy);
}
Also used : DynamicTableFactory(org.apache.flink.table.factories.DynamicTableFactory) IntStream(java.util.stream.IntStream) DeliveryGuarantee(org.apache.flink.connector.base.DeliveryGuarantee) DataType(org.apache.flink.table.types.DataType) FlinkException(org.apache.flink.util.FlinkException) ConfigOptions(org.apache.flink.configuration.ConfigOptions) Arrays(java.util.Arrays) SCAN_STARTUP_TIMESTAMP_MILLIS(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.SCAN_STARTUP_TIMESTAMP_MILLIS) KEY_FORMAT(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.KEY_FORMAT) TOPIC(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.TOPIC) TRANSACTIONAL_ID_PREFIX(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.TRANSACTIONAL_ID_PREFIX) DELIVERY_GUARANTEE(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.DELIVERY_GUARANTEE) TOPIC_PATTERN(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.TOPIC_PATTERN) HashMap(java.util.HashMap) VALUE_FIELDS_INCLUDE(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.VALUE_FIELDS_INCLUDE) InstantiationUtil(org.apache.flink.util.InstantiationUtil) ReadableConfig(org.apache.flink.configuration.ReadableConfig) Map(java.util.Map) ConfigOption(org.apache.flink.configuration.ConfigOption) FORMAT(org.apache.flink.table.factories.FactoryUtil.FORMAT) FlinkFixedPartitioner(org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner) SCAN_STARTUP_MODE(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.SCAN_STARTUP_MODE) RowData(org.apache.flink.table.data.RowData) Properties(java.util.Properties) KafkaTopicPartition(org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition) ValueFieldsStrategy(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.ValueFieldsStrategy) Configuration(org.apache.flink.configuration.Configuration) TableException(org.apache.flink.table.api.TableException) VALUE_FORMAT(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.VALUE_FORMAT) ScanStartupMode(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.ScanStartupMode) Preconditions(org.apache.flink.util.Preconditions) FlinkKafkaPartitioner(org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner) StartupMode(org.apache.flink.streaming.connectors.kafka.config.StartupMode) List(java.util.List) SINK_PARTITIONER(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.SINK_PARTITIONER) FactoryUtil(org.apache.flink.table.factories.FactoryUtil) LogicalType(org.apache.flink.table.types.logical.LogicalType) ValidationException(org.apache.flink.table.api.ValidationException) Optional(java.util.Optional) SCAN_STARTUP_SPECIFIC_OFFSETS(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.SCAN_STARTUP_SPECIFIC_OFFSETS) Internal(org.apache.flink.annotation.Internal) Pattern(java.util.regex.Pattern) KEY_FIELDS_PREFIX(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.KEY_FIELDS_PREFIX) LogicalTypeRoot(org.apache.flink.table.types.logical.LogicalTypeRoot) LogicalTypeChecks(org.apache.flink.table.types.logical.utils.LogicalTypeChecks) KEY_FIELDS(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.KEY_FIELDS) TableException(org.apache.flink.table.api.TableException) ValidationException(org.apache.flink.table.api.ValidationException) ValueFieldsStrategy(org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.ValueFieldsStrategy) LogicalType(org.apache.flink.table.types.logical.LogicalType) IntStream(java.util.stream.IntStream)

Aggregations

Arrays (java.util.Arrays)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 Properties (java.util.Properties)1 Pattern (java.util.regex.Pattern)1 IntStream (java.util.stream.IntStream)1 Internal (org.apache.flink.annotation.Internal)1 ConfigOption (org.apache.flink.configuration.ConfigOption)1 ConfigOptions (org.apache.flink.configuration.ConfigOptions)1 Configuration (org.apache.flink.configuration.Configuration)1 ReadableConfig (org.apache.flink.configuration.ReadableConfig)1 DeliveryGuarantee (org.apache.flink.connector.base.DeliveryGuarantee)1 StartupMode (org.apache.flink.streaming.connectors.kafka.config.StartupMode)1 KafkaTopicPartition (org.apache.flink.streaming.connectors.kafka.internals.KafkaTopicPartition)1 FlinkFixedPartitioner (org.apache.flink.streaming.connectors.kafka.partitioner.FlinkFixedPartitioner)1 FlinkKafkaPartitioner (org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner)1 DELIVERY_GUARANTEE (org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.DELIVERY_GUARANTEE)1 KEY_FIELDS (org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.KEY_FIELDS)1