Examples with ReadableConfig org.apache.flink.configuration.ReadableConfig used on opensource projects

Example 11 with ReadableConfig

use of org.apache.flink.configuration.ReadableConfig in project flink by apache.

the class KafkaDynamicTableFactory method createDynamicTableSource.

@Override
public DynamicTableSource createDynamicTableSource(Context context) {
    final TableFactoryHelper helper = FactoryUtil.createTableFactoryHelper(this, context);
    final Optional<DecodingFormat<DeserializationSchema<RowData>>> keyDecodingFormat = getKeyDecodingFormat(helper);
    final DecodingFormat<DeserializationSchema<RowData>> valueDecodingFormat = getValueDecodingFormat(helper);
    helper.validateExcept(PROPERTIES_PREFIX);
    final ReadableConfig tableOptions = helper.getOptions();
    validateTableSourceOptions(tableOptions);
    validatePKConstraints(context.getObjectIdentifier(), context.getPrimaryKeyIndexes(), context.getCatalogTable().getOptions(), valueDecodingFormat);
    final StartupOptions startupOptions = getStartupOptions(tableOptions);
    final Properties properties = getKafkaProperties(context.getCatalogTable().getOptions());
    // add topic-partition discovery
    final Optional<Long> partitionDiscoveryInterval = tableOptions.getOptional(SCAN_TOPIC_PARTITION_DISCOVERY).map(Duration::toMillis);
    properties.setProperty(KafkaSourceOptions.PARTITION_DISCOVERY_INTERVAL_MS.key(), partitionDiscoveryInterval.orElse(-1L).toString());
    final DataType physicalDataType = context.getPhysicalRowDataType();
    final int[] keyProjection = createKeyFormatProjection(tableOptions, physicalDataType);
    final int[] valueProjection = createValueFormatProjection(tableOptions, physicalDataType);
    final String keyPrefix = tableOptions.getOptional(KEY_FIELDS_PREFIX).orElse(null);
    return createKafkaTableSource(physicalDataType, keyDecodingFormat.orElse(null), valueDecodingFormat, keyProjection, valueProjection, keyPrefix, getSourceTopics(tableOptions), getSourceTopicPattern(tableOptions), properties, startupOptions.startupMode, startupOptions.specificOffsets, startupOptions.startupTimestampMillis, context.getObjectIdentifier().asSummaryString());
}

Example 12 with ReadableConfig

use of org.apache.flink.configuration.ReadableConfig in project flink by apache.

the class KafkaConnectorOptionsUtil method createValueFormatProjection.

/**
 * Creates an array of indices that determine which physical fields of the table schema to
 * include in the value format.
 *
 * <p>See {@link KafkaConnectorOptions#VALUE_FORMAT}, {@link
 * KafkaConnectorOptions#VALUE_FIELDS_INCLUDE}, and {@link
 * KafkaConnectorOptions#KEY_FIELDS_PREFIX} for more information.
 */
public static int[] createValueFormatProjection(ReadableConfig options, DataType physicalDataType) {
    final LogicalType physicalType = physicalDataType.getLogicalType();
    Preconditions.checkArgument(physicalType.is(LogicalTypeRoot.ROW), "Row data type expected.");
    final int physicalFieldCount = LogicalTypeChecks.getFieldCount(physicalType);
    final IntStream physicalFields = IntStream.range(0, physicalFieldCount);
    final String keyPrefix = options.getOptional(KEY_FIELDS_PREFIX).orElse("");
    final ValueFieldsStrategy strategy = options.get(VALUE_FIELDS_INCLUDE);
    if (strategy == ValueFieldsStrategy.ALL) {
        if (keyPrefix.length() > 0) {
            throw new ValidationException(String.format("A key prefix is not allowed when option '%s' is set to '%s'. " + "Set it to '%s' instead to avoid field overlaps.", VALUE_FIELDS_INCLUDE.key(), ValueFieldsStrategy.ALL, ValueFieldsStrategy.EXCEPT_KEY));
        }
        return physicalFields.toArray();
    } else if (strategy == ValueFieldsStrategy.EXCEPT_KEY) {
        final int[] keyProjection = createKeyFormatProjection(options, physicalDataType);
        return physicalFields.filter(pos -> IntStream.of(keyProjection).noneMatch(k -> k == pos)).toArray();
    }
    throw new TableException("Unknown value fields strategy:" + strategy);
}

Example 13 with ReadableConfig

use of org.apache.flink.configuration.ReadableConfig in project flink by apache.

the class StreamPhysicalPythonGroupWindowAggregateRule method convert.

@Override
public RelNode convert(RelNode rel) {
    FlinkLogicalWindowAggregate agg = (FlinkLogicalWindowAggregate) rel;
    LogicalWindow window = agg.getWindow();
    List<AggregateCall> aggCalls = agg.getAggCallList();
    boolean isPandasPythonUDAF = aggCalls.stream().anyMatch(x -> PythonUtil.isPythonAggregate(x, PythonFunctionKind.PANDAS));
    if (isPandasPythonUDAF && window instanceof SessionGroupWindow) {
        throw new TableException("Session Group Window is currently not supported for Pandas UDAF.");
    }
    RelNode input = agg.getInput();
    RelOptCluster cluster = rel.getCluster();
    FlinkRelDistribution requiredDistribution;
    if (agg.getGroupCount() != 0) {
        requiredDistribution = FlinkRelDistribution.hash(agg.getGroupSet().asList(), true);
    } else {
        requiredDistribution = FlinkRelDistribution.SINGLETON();
    }
    RelTraitSet requiredTraitSet = input.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL()).replace(requiredDistribution);
    RelTraitSet providedTraitSet = rel.getTraitSet().replace(FlinkConventions.STREAM_PHYSICAL());
    RelNode newInput = RelOptRule.convert(input, requiredTraitSet);
    ReadableConfig config = ShortcutUtils.unwrapTableConfig(rel);
    WindowEmitStrategy emitStrategy = WindowEmitStrategy.apply(config, agg.getWindow());
    if (emitStrategy.produceUpdates()) {
        throw new TableException("Python Group Window Aggregate Function is currently not supported for early fired or lately fired.");
    }
    return new StreamPhysicalPythonGroupWindowAggregate(cluster, providedTraitSet, newInput, rel.getRowType(), agg.getGroupSet().toArray(), JavaScalaConversionUtil.toScala(aggCalls), agg.getWindow(), agg.getNamedProperties(), emitStrategy);
}

Example 14 with ReadableConfig

use of org.apache.flink.configuration.ReadableConfig in project flink by apache.

the class CatalogSourceTable method computeContextResolvedTable.

private ContextResolvedTable computeContextResolvedTable(FlinkContext context, Map<String, String> hintedOptions) {
    ContextResolvedTable contextResolvedTable = schemaTable.getContextResolvedTable();
    if (hintedOptions.isEmpty()) {
        return contextResolvedTable;
    }
    final ReadableConfig config = context.getTableConfig().getConfiguration();
    if (!config.get(TableConfigOptions.TABLE_DYNAMIC_TABLE_OPTIONS_ENABLED)) {
        throw new ValidationException(String.format("The '%s' hint is allowed only when the config option '%s' is set to true.", FlinkHints.HINT_NAME_OPTIONS, TableConfigOptions.TABLE_DYNAMIC_TABLE_OPTIONS_ENABLED.key()));
    }
    if (contextResolvedTable.getResolvedTable().getTableKind() == TableKind.VIEW) {
        throw new ValidationException(String.format("View '%s' cannot be enriched with new options. " + "Hints can only be applied to tables.", contextResolvedTable.getIdentifier()));
    }
    return contextResolvedTable.copy(FlinkHints.mergeTableOptions(hintedOptions, contextResolvedTable.getResolvedTable().getOptions()));
}

Example 15 with ReadableConfig

use of org.apache.flink.configuration.ReadableConfig in project flink by apache.

the class LocalExecutor method executeQuery.

@Override
public ResultDescriptor executeQuery(String sessionId, QueryOperation query) throws SqlExecutionException {
    final TableResultInternal tableResult = executeOperation(sessionId, query);
    final SessionContext context = getSessionContext(sessionId);
    final ReadableConfig config = context.getReadableConfig();
    final DynamicResult result = resultStore.createResult(config, tableResult);
    checkArgument(tableResult.getJobClient().isPresent());
    String jobId = tableResult.getJobClient().get().getJobID().toString();
    // store the result under the JobID
    resultStore.storeResult(jobId, result);
    return new ResultDescriptor(jobId, tableResult.getResolvedSchema(), result.isMaterialized(), config, tableResult.getRowDataToStringConverter());
}