Search in sources :

Example 1 with ComputedColumn

use of org.apache.flink.table.catalog.Column.ComputedColumn in project flink by apache.

the class DynamicSourceUtils method pushGeneratedProjection.

/**
 * Creates a projection that adds computed columns and finalizes the table schema.
 */
private static void pushGeneratedProjection(FlinkRelBuilder relBuilder, ResolvedSchema schema) {
    final ExpressionConverter converter = new ExpressionConverter(relBuilder);
    final List<RexNode> projection = schema.getColumns().stream().map(c -> {
        if (c instanceof ComputedColumn) {
            final ComputedColumn computedColumn = (ComputedColumn) c;
            return computedColumn.getExpression().accept(converter);
        } else {
            return relBuilder.field(c.getName());
        }
    }).collect(Collectors.toList());
    relBuilder.projectNamed(projection, schema.getColumns().stream().map(Column::getName).collect(Collectors.toList()), true);
}
Also used : DataType(org.apache.flink.table.types.DataType) ScanRuntimeProvider(org.apache.flink.table.connector.source.ScanTableSource.ScanRuntimeProvider) ChangelogMode(org.apache.flink.table.connector.ChangelogMode) Column(org.apache.flink.table.catalog.Column) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) ShortcutUtils(org.apache.flink.table.planner.utils.ShortcutUtils) RowType(org.apache.flink.table.types.logical.RowType) ScanTableSource(org.apache.flink.table.connector.source.ScanTableSource) FlinkRelBuilder(org.apache.flink.table.planner.calcite.FlinkRelBuilder) MetadataColumn(org.apache.flink.table.catalog.Column.MetadataColumn) ReadableConfig(org.apache.flink.configuration.ReadableConfig) RexNode(org.apache.calcite.rex.RexNode) RowField(org.apache.flink.table.types.logical.RowType.RowField) RelHint(org.apache.calcite.rel.hint.RelHint) Map(java.util.Map) LogicalTypeCasts.supportsExplicitCast(org.apache.flink.table.types.logical.utils.LogicalTypeCasts.supportsExplicitCast) ResolvedCatalogTable(org.apache.flink.table.catalog.ResolvedCatalogTable) ContextResolvedTable(org.apache.flink.table.catalog.ContextResolvedTable) RelDataType(org.apache.calcite.rel.type.RelDataType) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) TableConfig(org.apache.flink.table.api.TableConfig) WatermarkSpec(org.apache.flink.table.catalog.WatermarkSpec) RexBuilder(org.apache.calcite.rex.RexBuilder) TableException(org.apache.flink.table.api.TableException) Set(java.util.Set) ExpressionConverter(org.apache.flink.table.planner.expressions.converter.ExpressionConverter) RelNode(org.apache.calcite.rel.RelNode) Collectors(java.util.stream.Collectors) SourceAbilitySpec(org.apache.flink.table.planner.plan.abilities.source.SourceAbilitySpec) TableSourceTable(org.apache.flink.table.planner.plan.schema.TableSourceTable) ComputedColumn(org.apache.flink.table.catalog.Column.ComputedColumn) DataStream(org.apache.flink.streaming.api.datastream.DataStream) List(java.util.List) Stream(java.util.stream.Stream) LogicalType(org.apache.flink.table.types.logical.LogicalType) FlinkStatistic(org.apache.flink.table.planner.plan.stats.FlinkStatistic) RowKind(org.apache.flink.types.RowKind) ValidationException(org.apache.flink.table.api.ValidationException) SupportsReadingMetadata(org.apache.flink.table.connector.source.abilities.SupportsReadingMetadata) Internal(org.apache.flink.annotation.Internal) TypeConversions(org.apache.flink.table.types.utils.TypeConversions) ExecutionConfigOptions(org.apache.flink.table.api.config.ExecutionConfigOptions) Collections(java.util.Collections) ScanRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext) LogicalTableScan(org.apache.calcite.rel.logical.LogicalTableScan) Column(org.apache.flink.table.catalog.Column) MetadataColumn(org.apache.flink.table.catalog.Column.MetadataColumn) ComputedColumn(org.apache.flink.table.catalog.Column.ComputedColumn) ExpressionConverter(org.apache.flink.table.planner.expressions.converter.ExpressionConverter) ComputedColumn(org.apache.flink.table.catalog.Column.ComputedColumn) RexNode(org.apache.calcite.rex.RexNode)

Example 2 with ComputedColumn

use of org.apache.flink.table.catalog.Column.ComputedColumn in project flink by apache.

the class DynamicSourceUtils method pushMetadataProjection.

/**
 * Creates a projection that reorders physical and metadata columns according to the given
 * schema. It casts metadata columns into the expected data type to be accessed by computed
 * columns in the next step. Computed columns are ignored here.
 *
 * @see SupportsReadingMetadata
 */
private static void pushMetadataProjection(FlinkRelBuilder relBuilder, ResolvedSchema schema) {
    final RexBuilder rexBuilder = relBuilder.getRexBuilder();
    final List<String> fieldNames = schema.getColumns().stream().filter(c -> !(c instanceof ComputedColumn)).map(Column::getName).collect(Collectors.toList());
    final List<RexNode> fieldNodes = schema.getColumns().stream().filter(c -> !(c instanceof ComputedColumn)).map(c -> {
        final RelDataType relDataType = relBuilder.getTypeFactory().createFieldTypeFromLogicalType(c.getDataType().getLogicalType());
        if (c instanceof MetadataColumn) {
            final MetadataColumn metadataColumn = (MetadataColumn) c;
            final String metadataKey = metadataColumn.getMetadataKey().orElse(metadataColumn.getName());
            return rexBuilder.makeAbstractCast(relDataType, relBuilder.field(metadataKey));
        } else {
            return relBuilder.field(c.getName());
        }
    }).collect(Collectors.toList());
    relBuilder.projectNamed(fieldNodes, fieldNames, true);
}
Also used : DataType(org.apache.flink.table.types.DataType) ScanRuntimeProvider(org.apache.flink.table.connector.source.ScanTableSource.ScanRuntimeProvider) ChangelogMode(org.apache.flink.table.connector.ChangelogMode) Column(org.apache.flink.table.catalog.Column) ResolvedSchema(org.apache.flink.table.catalog.ResolvedSchema) ShortcutUtils(org.apache.flink.table.planner.utils.ShortcutUtils) RowType(org.apache.flink.table.types.logical.RowType) ScanTableSource(org.apache.flink.table.connector.source.ScanTableSource) FlinkRelBuilder(org.apache.flink.table.planner.calcite.FlinkRelBuilder) MetadataColumn(org.apache.flink.table.catalog.Column.MetadataColumn) ReadableConfig(org.apache.flink.configuration.ReadableConfig) RexNode(org.apache.calcite.rex.RexNode) RowField(org.apache.flink.table.types.logical.RowType.RowField) RelHint(org.apache.calcite.rel.hint.RelHint) Map(java.util.Map) LogicalTypeCasts.supportsExplicitCast(org.apache.flink.table.types.logical.utils.LogicalTypeCasts.supportsExplicitCast) ResolvedCatalogTable(org.apache.flink.table.catalog.ResolvedCatalogTable) ContextResolvedTable(org.apache.flink.table.catalog.ContextResolvedTable) RelDataType(org.apache.calcite.rel.type.RelDataType) DynamicTableSource(org.apache.flink.table.connector.source.DynamicTableSource) TableConfig(org.apache.flink.table.api.TableConfig) WatermarkSpec(org.apache.flink.table.catalog.WatermarkSpec) RexBuilder(org.apache.calcite.rex.RexBuilder) TableException(org.apache.flink.table.api.TableException) Set(java.util.Set) ExpressionConverter(org.apache.flink.table.planner.expressions.converter.ExpressionConverter) RelNode(org.apache.calcite.rel.RelNode) Collectors(java.util.stream.Collectors) SourceAbilitySpec(org.apache.flink.table.planner.plan.abilities.source.SourceAbilitySpec) TableSourceTable(org.apache.flink.table.planner.plan.schema.TableSourceTable) ComputedColumn(org.apache.flink.table.catalog.Column.ComputedColumn) DataStream(org.apache.flink.streaming.api.datastream.DataStream) List(java.util.List) Stream(java.util.stream.Stream) LogicalType(org.apache.flink.table.types.logical.LogicalType) FlinkStatistic(org.apache.flink.table.planner.plan.stats.FlinkStatistic) RowKind(org.apache.flink.types.RowKind) ValidationException(org.apache.flink.table.api.ValidationException) SupportsReadingMetadata(org.apache.flink.table.connector.source.abilities.SupportsReadingMetadata) Internal(org.apache.flink.annotation.Internal) TypeConversions(org.apache.flink.table.types.utils.TypeConversions) ExecutionConfigOptions(org.apache.flink.table.api.config.ExecutionConfigOptions) Collections(java.util.Collections) ScanRuntimeProviderContext(org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext) LogicalTableScan(org.apache.calcite.rel.logical.LogicalTableScan) MetadataColumn(org.apache.flink.table.catalog.Column.MetadataColumn) RexBuilder(org.apache.calcite.rex.RexBuilder) ComputedColumn(org.apache.flink.table.catalog.Column.ComputedColumn) RelDataType(org.apache.calcite.rel.type.RelDataType) RexNode(org.apache.calcite.rex.RexNode)

Aggregations

Collections (java.util.Collections)2 List (java.util.List)2 Map (java.util.Map)2 Set (java.util.Set)2 Collectors (java.util.stream.Collectors)2 Stream (java.util.stream.Stream)2 RelNode (org.apache.calcite.rel.RelNode)2 RelHint (org.apache.calcite.rel.hint.RelHint)2 LogicalTableScan (org.apache.calcite.rel.logical.LogicalTableScan)2 RelDataType (org.apache.calcite.rel.type.RelDataType)2 RexBuilder (org.apache.calcite.rex.RexBuilder)2 RexNode (org.apache.calcite.rex.RexNode)2 Internal (org.apache.flink.annotation.Internal)2 ReadableConfig (org.apache.flink.configuration.ReadableConfig)2 DataStream (org.apache.flink.streaming.api.datastream.DataStream)2 TableConfig (org.apache.flink.table.api.TableConfig)2 TableException (org.apache.flink.table.api.TableException)2 ValidationException (org.apache.flink.table.api.ValidationException)2 ExecutionConfigOptions (org.apache.flink.table.api.config.ExecutionConfigOptions)2 Column (org.apache.flink.table.catalog.Column)2