Examples with SourceWatermarkSpec org.apache.flink.table.planner.plan.abilities.source.SourceWatermarkSpec used on opensource projects

Example 1 with SourceWatermarkSpec

use of org.apache.flink.table.planner.plan.abilities.source.SourceWatermarkSpec in project flink by apache.

the class PushWatermarkIntoTableSourceScanRuleBase method getNewScan.

/**
 * It uses the input watermark expression to generate the {@link WatermarkGeneratorSupplier}.
 * After the {@link WatermarkStrategy} is pushed into the scan, it will build a new scan.
 * However, when {@link FlinkLogicalWatermarkAssigner} is the parent of the {@link
 * FlinkLogicalTableSourceScan} it should modify the rowtime type to keep the type of plan is
 * consistent. In other cases, it just keep the data type of the scan as same as before and
 * leave the work when rewriting the projection.
 *
 * <p>NOTES: the row type of the scan is not always as same as the watermark assigner. Because
 * the scan will not add the rowtime column into the row when pushing the watermark assigner
 * into the scan. In some cases, query may have computed columns defined on rowtime column. If
 * modifying the type of the rowtime(with time attribute), it will also influence the type of
 * the computed column. Therefore, if the watermark assigner is not the parent of the scan, set
 * the type of the scan as before and leave the work to projection.
 */
protected FlinkLogicalTableSourceScan getNewScan(FlinkLogicalWatermarkAssigner watermarkAssigner, RexNode watermarkExpr, FlinkLogicalTableSourceScan scan, TableConfig tableConfig, boolean useWatermarkAssignerRowType) {
    final TableSourceTable tableSourceTable = scan.getTable().unwrap(TableSourceTable.class);
    final DynamicTableSource newDynamicTableSource = tableSourceTable.tableSource().copy();
    final boolean isSourceWatermark = newDynamicTableSource instanceof SupportsSourceWatermark && hasSourceWatermarkDeclaration(watermarkExpr);
    final RelDataType newType;
    if (useWatermarkAssignerRowType) {
        // project is trivial and set rowtime type in scan
        newType = watermarkAssigner.getRowType();
    } else {
        // project add/delete columns and set the rowtime column type in project
        newType = scan.getRowType();
    }
    final RowType producedType = (RowType) FlinkTypeFactory.toLogicalType(newType);
    final SourceAbilityContext abilityContext = SourceAbilityContext.from(scan);
    final SourceAbilitySpec abilitySpec;
    if (isSourceWatermark) {
        final SourceWatermarkSpec sourceWatermarkSpec = new SourceWatermarkSpec(true, producedType);
        sourceWatermarkSpec.apply(newDynamicTableSource, abilityContext);
        abilitySpec = sourceWatermarkSpec;
    } else {
        final Duration idleTimeout = tableConfig.getConfiguration().get(ExecutionConfigOptions.TABLE_EXEC_SOURCE_IDLE_TIMEOUT);
        final long idleTimeoutMillis;
        if (!idleTimeout.isZero() && !idleTimeout.isNegative()) {
            idleTimeoutMillis = idleTimeout.toMillis();
        } else {
            idleTimeoutMillis = -1L;
        }
        final WatermarkPushDownSpec watermarkPushDownSpec = new WatermarkPushDownSpec(watermarkExpr, idleTimeoutMillis, producedType);
        watermarkPushDownSpec.apply(newDynamicTableSource, abilityContext);
        abilitySpec = watermarkPushDownSpec;
    }
    TableSourceTable newTableSourceTable = tableSourceTable.copy(newDynamicTableSource, newType, new SourceAbilitySpec[] { abilitySpec });
    return FlinkLogicalTableSourceScan.create(scan.getCluster(), scan.getHints(), newTableSourceTable);
}

Example 2 with SourceWatermarkSpec

use of org.apache.flink.table.planner.plan.abilities.source.SourceWatermarkSpec in project flink by apache.

the class DynamicTableSourceSpecSerdeTest method testDynamicTableSinkSpecSerde.

public static Stream<DynamicTableSourceSpec> testDynamicTableSinkSpecSerde() {
    Map<String, String> options1 = new HashMap<>();
    options1.put("connector", FileSystemTableFactory.IDENTIFIER);
    options1.put("format", TestCsvFormatFactory.IDENTIFIER);
    options1.put("path", "/tmp");
    final ResolvedSchema resolvedSchema1 = new ResolvedSchema(Collections.singletonList(Column.physical("a", DataTypes.BIGINT())), Collections.emptyList(), null);
    final CatalogTable catalogTable1 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema1).build(), null, Collections.emptyList(), options1);
    DynamicTableSourceSpec spec1 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable1, resolvedSchema1)), null);
    Map<String, String> options2 = new HashMap<>();
    options2.put("connector", TestValuesTableFactory.IDENTIFIER);
    options2.put("disable-lookup", "true");
    options2.put("enable-watermark-push-down", "true");
    options2.put("filterable-fields", "b");
    options2.put("bounded", "false");
    options2.put("readable-metadata", "m1:INT, m2:STRING");
    final ResolvedSchema resolvedSchema2 = new ResolvedSchema(Arrays.asList(Column.physical("a", DataTypes.BIGINT()), Column.physical("b", DataTypes.INT()), Column.physical("c", DataTypes.STRING()), Column.physical("p", DataTypes.STRING()), Column.metadata("m1", DataTypes.INT(), null, false), Column.metadata("m2", DataTypes.STRING(), null, false), Column.physical("ts", DataTypes.TIMESTAMP(3))), Collections.emptyList(), null);
    final CatalogTable catalogTable2 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema2).build(), null, Collections.emptyList(), options2);
    FlinkTypeFactory factory = FlinkTypeFactory.INSTANCE();
    RexBuilder rexBuilder = new RexBuilder(factory);
    DynamicTableSourceSpec spec2 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable2, resolvedSchema2)), Arrays.asList(new ProjectPushDownSpec(new int[][] { { 0 }, { 1 }, { 4 }, { 6 } }, RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new ReadingMetadataSpec(Arrays.asList("m1", "m2"), RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new FilterPushDownSpec(Collections.singletonList(// b >= 10
    rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.INTEGER), 1), rexBuilder.makeExactLiteral(new BigDecimal(10))))), new WatermarkPushDownSpec(rexBuilder.makeCall(SqlStdOperatorTable.MINUS, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.TIMESTAMP, 3), 3), rexBuilder.makeIntervalLiteral(BigDecimal.valueOf(1000), new SqlIntervalQualifier(TimeUnit.SECOND, 2, TimeUnit.SECOND, 6, SqlParserPos.ZERO))), 5000, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new SourceWatermarkSpec(true, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new LimitPushDownSpec(100), new PartitionPushDownSpec(Arrays.asList(new HashMap<String, String>() {

        {
            put("p", "A");
        }
    }, new HashMap<String, String>() {

        {
            put("p", "B");
        }
    }))));
    return Stream.of(spec1, spec2);
}