Examples with KEY_FIELDS_PREFIX org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.KEY_FIELDS_PREFIX used on opensource projects

Example 1 with KEY_FIELDS_PREFIX

use of org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.KEY_FIELDS_PREFIX in project flink by apache.

the class KafkaConnectorOptionsUtil method createValueFormatProjection.

/**
 * Creates an array of indices that determine which physical fields of the table schema to
 * include in the value format.
 *
 * <p>See {@link KafkaConnectorOptions#VALUE_FORMAT}, {@link
 * KafkaConnectorOptions#VALUE_FIELDS_INCLUDE}, and {@link
 * KafkaConnectorOptions#KEY_FIELDS_PREFIX} for more information.
 */
public static int[] createValueFormatProjection(ReadableConfig options, DataType physicalDataType) {
    final LogicalType physicalType = physicalDataType.getLogicalType();
    Preconditions.checkArgument(physicalType.is(LogicalTypeRoot.ROW), "Row data type expected.");
    final int physicalFieldCount = LogicalTypeChecks.getFieldCount(physicalType);
    final IntStream physicalFields = IntStream.range(0, physicalFieldCount);
    final String keyPrefix = options.getOptional(KEY_FIELDS_PREFIX).orElse("");
    final ValueFieldsStrategy strategy = options.get(VALUE_FIELDS_INCLUDE);
    if (strategy == ValueFieldsStrategy.ALL) {
        if (keyPrefix.length() > 0) {
            throw new ValidationException(String.format("A key prefix is not allowed when option '%s' is set to '%s'. " + "Set it to '%s' instead to avoid field overlaps.", VALUE_FIELDS_INCLUDE.key(), ValueFieldsStrategy.ALL, ValueFieldsStrategy.EXCEPT_KEY));
        }
        return physicalFields.toArray();
    } else if (strategy == ValueFieldsStrategy.EXCEPT_KEY) {
        final int[] keyProjection = createKeyFormatProjection(options, physicalDataType);
        return physicalFields.filter(pos -> IntStream.of(keyProjection).noneMatch(k -> k == pos)).toArray();
    }
    throw new TableException("Unknown value fields strategy:" + strategy);
}

Example 2 with KEY_FIELDS_PREFIX

use of org.apache.flink.streaming.connectors.kafka.table.KafkaConnectorOptions.KEY_FIELDS_PREFIX in project flink by apache.

the class KafkaConnectorOptionsUtil method createKeyFormatProjection.

/**
 * Creates an array of indices that determine which physical fields of the table schema to
 * include in the key format and the order that those fields have in the key format.
 *
 * <p>See {@link KafkaConnectorOptions#KEY_FORMAT}, {@link KafkaConnectorOptions#KEY_FIELDS},
 * and {@link KafkaConnectorOptions#KEY_FIELDS_PREFIX} for more information.
 */
public static int[] createKeyFormatProjection(ReadableConfig options, DataType physicalDataType) {
    final LogicalType physicalType = physicalDataType.getLogicalType();
    Preconditions.checkArgument(physicalType.is(LogicalTypeRoot.ROW), "Row data type expected.");
    final Optional<String> optionalKeyFormat = options.getOptional(KEY_FORMAT);
    final Optional<List<String>> optionalKeyFields = options.getOptional(KEY_FIELDS);
    if (!optionalKeyFormat.isPresent() && optionalKeyFields.isPresent()) {
        throw new ValidationException(String.format("The option '%s' can only be declared if a key format is defined using '%s'.", KEY_FIELDS.key(), KEY_FORMAT.key()));
    } else if (optionalKeyFormat.isPresent() && (!optionalKeyFields.isPresent() || optionalKeyFields.get().size() == 0)) {
        throw new ValidationException(String.format("A key format '%s' requires the declaration of one or more of key fields using '%s'.", KEY_FORMAT.key(), KEY_FIELDS.key()));
    }
    if (!optionalKeyFormat.isPresent()) {
        return new int[0];
    }
    final String keyPrefix = options.getOptional(KEY_FIELDS_PREFIX).orElse("");
    final List<String> keyFields = optionalKeyFields.get();
    final List<String> physicalFields = LogicalTypeChecks.getFieldNames(physicalType);
    return keyFields.stream().mapToInt(keyField -> {
        final int pos = physicalFields.indexOf(keyField);
        // check that field name exists
        if (pos < 0) {
            throw new ValidationException(String.format("Could not find the field '%s' in the table schema for usage in the key format. " + "A key field must be a regular, physical column. " + "The following columns can be selected in the '%s' option:\n" + "%s", keyField, KEY_FIELDS.key(), physicalFields));
        }
        // check that field name is prefixed correctly
        if (!keyField.startsWith(keyPrefix)) {
            throw new ValidationException(String.format("All fields in '%s' must be prefixed with '%s' when option '%s' " + "is set but field '%s' is not prefixed.", KEY_FIELDS.key(), keyPrefix, KEY_FIELDS_PREFIX.key(), keyField));
        }
        return pos;
    }).toArray();
}