Examples with WatermarkStrategy org.apache.flink.table.sources.wmstrategies.WatermarkStrategy used on opensource projects

Example 1 with WatermarkStrategy

use of org.apache.flink.table.sources.wmstrategies.WatermarkStrategy in project flink by apache.

the class StreamExecLegacyTableSourceScan method createConversionTransformationIfNeeded.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> createConversionTransformationIfNeeded(StreamExecutionEnvironment streamExecEnv, ExecNodeConfig config, Transformation<?> sourceTransform, @Nullable RexNode rowtimeExpression) {
    final RowType outputType = (RowType) getOutputType();
    final Transformation<RowData> transformation;
    final int[] fieldIndexes = computeIndexMapping(true);
    if (needInternalConversion(fieldIndexes)) {
        final String extractElement, resetElement;
        if (ScanUtil.hasTimeAttributeField(fieldIndexes)) {
            String elementTerm = OperatorCodeGenerator.ELEMENT();
            extractElement = String.format("ctx.%s = %s;", elementTerm, elementTerm);
            resetElement = String.format("ctx.%s = null;", elementTerm);
        } else {
            extractElement = "";
            resetElement = "";
        }
        final CodeGeneratorContext ctx = new CodeGeneratorContext(config.getTableConfig()).setOperatorBaseClass(TableStreamOperator.class);
        // the produced type may not carry the correct precision user defined in DDL, because
        // it may be converted from legacy type. Fix precision using logical schema from DDL.
        // Code generation requires the correct precision of input fields.
        final DataType fixedProducedDataType = TableSourceUtil.fixPrecisionForProducedDataType(tableSource, outputType);
        transformation = ScanUtil.convertToInternalRow(ctx, (Transformation<Object>) sourceTransform, fieldIndexes, fixedProducedDataType, outputType, qualifiedName, (detailName, simplifyName) -> createFormattedTransformationName(detailName, simplifyName, config), (description) -> createFormattedTransformationDescription(description, config), JavaScalaConversionUtil.toScala(Optional.ofNullable(rowtimeExpression)), extractElement, resetElement);
    } else {
        transformation = (Transformation<RowData>) sourceTransform;
    }
    final RelDataType relDataType = FlinkTypeFactory.INSTANCE().buildRelNodeRowType(outputType);
    final DataStream<RowData> ingestedTable = new DataStream<>(streamExecEnv, transformation);
    final Optional<RowtimeAttributeDescriptor> rowtimeDesc = JavaScalaConversionUtil.toJava(TableSourceUtil.getRowtimeAttributeDescriptor(tableSource, relDataType));
    final DataStream<RowData> withWatermarks = rowtimeDesc.map(desc -> {
        int rowtimeFieldIdx = relDataType.getFieldNames().indexOf(desc.getAttributeName());
        WatermarkStrategy strategy = desc.getWatermarkStrategy();
        if (strategy instanceof PeriodicWatermarkAssigner) {
            PeriodicWatermarkAssignerWrapper watermarkGenerator = new PeriodicWatermarkAssignerWrapper((PeriodicWatermarkAssigner) strategy, rowtimeFieldIdx);
            return ingestedTable.assignTimestampsAndWatermarks(watermarkGenerator);
        } else if (strategy instanceof PunctuatedWatermarkAssigner) {
            PunctuatedWatermarkAssignerWrapper watermarkGenerator = new PunctuatedWatermarkAssignerWrapper((PunctuatedWatermarkAssigner) strategy, rowtimeFieldIdx, tableSource.getProducedDataType());
            return ingestedTable.assignTimestampsAndWatermarks(watermarkGenerator);
        } else {
            // underlying DataStream.
            return ingestedTable;
        }
    }).orElse(// No need to generate watermarks if no rowtime
    ingestedTable);
    // attribute is specified.
    return withWatermarks.getTransformation();
}

Example 2 with WatermarkStrategy

use of org.apache.flink.table.sources.wmstrategies.WatermarkStrategy in project flink by apache.

the class RowtimeValidator method getRowtimeComponents.

// utilities
public static Optional<Tuple2<TimestampExtractor, WatermarkStrategy>> getRowtimeComponents(DescriptorProperties properties, String prefix) {
    // create timestamp extractor
    TimestampExtractor extractor;
    Optional<String> t = properties.getOptionalString(prefix + ROWTIME_TIMESTAMPS_TYPE);
    if (!t.isPresent()) {
        return Optional.empty();
    }
    switch(t.get()) {
        case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD:
            String field = properties.getString(prefix + ROWTIME_TIMESTAMPS_FROM);
            extractor = new ExistingField(field);
            break;
        case ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE:
            extractor = StreamRecordTimestamp.INSTANCE;
            break;
        case ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM:
            Class<TimestampExtractor> clazz = properties.getClass(prefix + ROWTIME_TIMESTAMPS_CLASS, TimestampExtractor.class);
            extractor = EncodingUtils.decodeStringToObject(properties.getString(prefix + ROWTIME_TIMESTAMPS_SERIALIZED), clazz);
            break;
        default:
            throw new ValidationException("Unsupported rowtime timestamps type: " + t.get());
    }
    // create watermark strategy
    WatermarkStrategy strategy;
    String s = properties.getString(prefix + ROWTIME_WATERMARKS_TYPE);
    switch(s) {
        case ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING:
            strategy = new AscendingTimestamps();
            break;
        case ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED:
            long delay = properties.getLong(prefix + ROWTIME_WATERMARKS_DELAY);
            strategy = new BoundedOutOfOrderTimestamps(delay);
            break;
        case ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE:
            strategy = PreserveWatermarks.INSTANCE;
            break;
        case ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM:
            Class<WatermarkStrategy> clazz = properties.getClass(prefix + ROWTIME_WATERMARKS_CLASS, WatermarkStrategy.class);
            strategy = EncodingUtils.decodeStringToObject(properties.getString(prefix + ROWTIME_WATERMARKS_SERIALIZED), clazz);
            break;
        default:
            throw new RuntimeException("Unsupported rowtime timestamps type: " + s);
    }
    return Optional.of(new Tuple2<>(extractor, strategy));
}