Search in sources :

Example 1 with PeriodicWatermarkAssigner

use of org.apache.flink.table.sources.wmstrategies.PeriodicWatermarkAssigner in project flink by apache.

the class StreamExecLegacyTableSourceScan method createConversionTransformationIfNeeded.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> createConversionTransformationIfNeeded(StreamExecutionEnvironment streamExecEnv, ExecNodeConfig config, Transformation<?> sourceTransform, @Nullable RexNode rowtimeExpression) {
    final RowType outputType = (RowType) getOutputType();
    final Transformation<RowData> transformation;
    final int[] fieldIndexes = computeIndexMapping(true);
    if (needInternalConversion(fieldIndexes)) {
        final String extractElement, resetElement;
        if (ScanUtil.hasTimeAttributeField(fieldIndexes)) {
            String elementTerm = OperatorCodeGenerator.ELEMENT();
            extractElement = String.format("ctx.%s = %s;", elementTerm, elementTerm);
            resetElement = String.format("ctx.%s = null;", elementTerm);
        } else {
            extractElement = "";
            resetElement = "";
        }
        final CodeGeneratorContext ctx = new CodeGeneratorContext(config.getTableConfig()).setOperatorBaseClass(TableStreamOperator.class);
        // the produced type may not carry the correct precision user defined in DDL, because
        // it may be converted from legacy type. Fix precision using logical schema from DDL.
        // Code generation requires the correct precision of input fields.
        final DataType fixedProducedDataType = TableSourceUtil.fixPrecisionForProducedDataType(tableSource, outputType);
        transformation = ScanUtil.convertToInternalRow(ctx, (Transformation<Object>) sourceTransform, fieldIndexes, fixedProducedDataType, outputType, qualifiedName, (detailName, simplifyName) -> createFormattedTransformationName(detailName, simplifyName, config), (description) -> createFormattedTransformationDescription(description, config), JavaScalaConversionUtil.toScala(Optional.ofNullable(rowtimeExpression)), extractElement, resetElement);
    } else {
        transformation = (Transformation<RowData>) sourceTransform;
    }
    final RelDataType relDataType = FlinkTypeFactory.INSTANCE().buildRelNodeRowType(outputType);
    final DataStream<RowData> ingestedTable = new DataStream<>(streamExecEnv, transformation);
    final Optional<RowtimeAttributeDescriptor> rowtimeDesc = JavaScalaConversionUtil.toJava(TableSourceUtil.getRowtimeAttributeDescriptor(tableSource, relDataType));
    final DataStream<RowData> withWatermarks = rowtimeDesc.map(desc -> {
        int rowtimeFieldIdx = relDataType.getFieldNames().indexOf(desc.getAttributeName());
        WatermarkStrategy strategy = desc.getWatermarkStrategy();
        if (strategy instanceof PeriodicWatermarkAssigner) {
            PeriodicWatermarkAssignerWrapper watermarkGenerator = new PeriodicWatermarkAssignerWrapper((PeriodicWatermarkAssigner) strategy, rowtimeFieldIdx);
            return ingestedTable.assignTimestampsAndWatermarks(watermarkGenerator);
        } else if (strategy instanceof PunctuatedWatermarkAssigner) {
            PunctuatedWatermarkAssignerWrapper watermarkGenerator = new PunctuatedWatermarkAssignerWrapper((PunctuatedWatermarkAssigner) strategy, rowtimeFieldIdx, tableSource.getProducedDataType());
            return ingestedTable.assignTimestampsAndWatermarks(watermarkGenerator);
        } else {
            // underlying DataStream.
            return ingestedTable;
        }
    }).orElse(// No need to generate watermarks if no rowtime
    ingestedTable);
    // attribute is specified.
    return withWatermarks.getTransformation();
}
Also used : TableStreamOperator(org.apache.flink.table.runtime.operators.TableStreamOperator) DataType(org.apache.flink.table.types.DataType) TableSourceUtil(org.apache.flink.table.planner.sources.TableSourceUtil) RowtimeAttributeDescriptor(org.apache.flink.table.sources.RowtimeAttributeDescriptor) TableSource(org.apache.flink.table.sources.TableSource) PeriodicWatermarkAssigner(org.apache.flink.table.sources.wmstrategies.PeriodicWatermarkAssigner) FlinkTypeFactory(org.apache.flink.table.planner.calcite.FlinkTypeFactory) RowType(org.apache.flink.table.types.logical.RowType) ExecNode(org.apache.flink.table.planner.plan.nodes.exec.ExecNode) ScanUtil(org.apache.flink.table.planner.plan.utils.ScanUtil) RexNode(org.apache.calcite.rex.RexNode) InputFormat(org.apache.flink.api.common.io.InputFormat) TypeInformation(org.apache.flink.api.common.typeinfo.TypeInformation) CodeGeneratorContext(org.apache.flink.table.planner.codegen.CodeGeneratorContext) Nullable(javax.annotation.Nullable) RelDataType(org.apache.calcite.rel.type.RelDataType) ExecNodeContext(org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext) RowData(org.apache.flink.table.data.RowData) InputSplit(org.apache.flink.core.io.InputSplit) ExecNodeConfig(org.apache.flink.table.planner.plan.nodes.exec.ExecNodeConfig) WatermarkStrategy(org.apache.flink.table.sources.wmstrategies.WatermarkStrategy) PunctuatedWatermarkAssigner(org.apache.flink.table.sources.wmstrategies.PunctuatedWatermarkAssigner) StreamTableSource(org.apache.flink.table.sources.StreamTableSource) DataStream(org.apache.flink.streaming.api.datastream.DataStream) OperatorCodeGenerator(org.apache.flink.table.planner.codegen.OperatorCodeGenerator) CommonExecLegacyTableSourceScan(org.apache.flink.table.planner.plan.nodes.exec.common.CommonExecLegacyTableSourceScan) List(java.util.List) PunctuatedWatermarkAssignerWrapper(org.apache.flink.table.runtime.operators.wmassigners.PunctuatedWatermarkAssignerWrapper) JavaScalaConversionUtil(org.apache.flink.table.planner.utils.JavaScalaConversionUtil) Optional(java.util.Optional) Transformation(org.apache.flink.api.dag.Transformation) PeriodicWatermarkAssignerWrapper(org.apache.flink.table.runtime.operators.wmassigners.PeriodicWatermarkAssignerWrapper) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) Transformation(org.apache.flink.api.dag.Transformation) CodeGeneratorContext(org.apache.flink.table.planner.codegen.CodeGeneratorContext) DataStream(org.apache.flink.streaming.api.datastream.DataStream) RowType(org.apache.flink.table.types.logical.RowType) RelDataType(org.apache.calcite.rel.type.RelDataType) PeriodicWatermarkAssigner(org.apache.flink.table.sources.wmstrategies.PeriodicWatermarkAssigner) RowData(org.apache.flink.table.data.RowData) PunctuatedWatermarkAssignerWrapper(org.apache.flink.table.runtime.operators.wmassigners.PunctuatedWatermarkAssignerWrapper) RowtimeAttributeDescriptor(org.apache.flink.table.sources.RowtimeAttributeDescriptor) PunctuatedWatermarkAssigner(org.apache.flink.table.sources.wmstrategies.PunctuatedWatermarkAssigner) DataType(org.apache.flink.table.types.DataType) RelDataType(org.apache.calcite.rel.type.RelDataType) WatermarkStrategy(org.apache.flink.table.sources.wmstrategies.WatermarkStrategy) PeriodicWatermarkAssignerWrapper(org.apache.flink.table.runtime.operators.wmassigners.PeriodicWatermarkAssignerWrapper)

Aggregations

List (java.util.List)1 Optional (java.util.Optional)1 Nullable (javax.annotation.Nullable)1 RelDataType (org.apache.calcite.rel.type.RelDataType)1 RexNode (org.apache.calcite.rex.RexNode)1 InputFormat (org.apache.flink.api.common.io.InputFormat)1 TypeInformation (org.apache.flink.api.common.typeinfo.TypeInformation)1 Transformation (org.apache.flink.api.dag.Transformation)1 InputSplit (org.apache.flink.core.io.InputSplit)1 DataStream (org.apache.flink.streaming.api.datastream.DataStream)1 StreamExecutionEnvironment (org.apache.flink.streaming.api.environment.StreamExecutionEnvironment)1 RowData (org.apache.flink.table.data.RowData)1 FlinkTypeFactory (org.apache.flink.table.planner.calcite.FlinkTypeFactory)1 CodeGeneratorContext (org.apache.flink.table.planner.codegen.CodeGeneratorContext)1 OperatorCodeGenerator (org.apache.flink.table.planner.codegen.OperatorCodeGenerator)1 ExecNode (org.apache.flink.table.planner.plan.nodes.exec.ExecNode)1 ExecNodeConfig (org.apache.flink.table.planner.plan.nodes.exec.ExecNodeConfig)1 ExecNodeContext (org.apache.flink.table.planner.plan.nodes.exec.ExecNodeContext)1 CommonExecLegacyTableSourceScan (org.apache.flink.table.planner.plan.nodes.exec.common.CommonExecLegacyTableSourceScan)1 ScanUtil (org.apache.flink.table.planner.plan.utils.ScanUtil)1