Examples with LimitPushDownSpec org.apache.flink.table.planner.plan.abilities.source.LimitPushDownSpec used on opensource projects

Example 1 with LimitPushDownSpec

use of org.apache.flink.table.planner.plan.abilities.source.LimitPushDownSpec in project flink by apache.

the class TemporalTableSourceSpecSerdeTest method testTemporalTableSourceSpecSerde.

public static Stream<TemporalTableSourceSpec> testTemporalTableSourceSpecSerde() {
    Map<String, String> options1 = new HashMap<>();
    options1.put("connector", "filesystem");
    options1.put("format", "testcsv");
    options1.put("path", "/tmp");
    final ResolvedSchema resolvedSchema1 = new ResolvedSchema(Collections.singletonList(Column.physical("a", DataTypes.BIGINT())), Collections.emptyList(), null);
    final CatalogTable catalogTable1 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema1).build(), null, Collections.emptyList(), options1);
    ResolvedCatalogTable resolvedCatalogTable = new ResolvedCatalogTable(catalogTable1, resolvedSchema1);
    RelDataType relDataType1 = FACTORY.createSqlType(SqlTypeName.BIGINT);
    LookupTableSource lookupTableSource = new TestValuesTableFactory.MockedLookupTableSource();
    TableSourceTable tableSourceTable1 = new TableSourceTable(null, relDataType1, FlinkStatistic.UNKNOWN(), lookupTableSource, true, ContextResolvedTable.temporary(ObjectIdentifier.of("default_catalog", "default_db", "MyTable"), resolvedCatalogTable), FLINK_CONTEXT, new SourceAbilitySpec[] { new LimitPushDownSpec(100) });
    TemporalTableSourceSpec temporalTableSourceSpec1 = new TemporalTableSourceSpec(tableSourceTable1);
    return Stream.of(temporalTableSourceSpec1);
}

Example 2 with LimitPushDownSpec

use of org.apache.flink.table.planner.plan.abilities.source.LimitPushDownSpec in project flink by apache.

the class PushLimitIntoTableSourceScanRule method applyLimit.

private TableSourceTable applyLimit(long limit, FlinkLogicalTableSourceScan scan) {
    TableSourceTable relOptTable = scan.getTable().unwrap(TableSourceTable.class);
    TableSourceTable oldTableSourceTable = relOptTable.unwrap(TableSourceTable.class);
    DynamicTableSource newTableSource = oldTableSourceTable.tableSource().copy();
    LimitPushDownSpec limitPushDownSpec = new LimitPushDownSpec(limit);
    limitPushDownSpec.apply(newTableSource, SourceAbilityContext.from(scan));
    FlinkStatistic statistic = relOptTable.getStatistic();
    final long newRowCount;
    if (statistic.getRowCount() != null) {
        newRowCount = Math.min(limit, statistic.getRowCount().longValue());
    } else {
        newRowCount = limit;
    }
    // update TableStats after limit push down
    TableStats newTableStats = new TableStats(newRowCount);
    FlinkStatistic newStatistic = FlinkStatistic.builder().statistic(statistic).tableStats(newTableStats).build();
    return oldTableSourceTable.copy(newTableSource, newStatistic, new SourceAbilitySpec[] { limitPushDownSpec });
}

Example 3 with LimitPushDownSpec

use of org.apache.flink.table.planner.plan.abilities.source.LimitPushDownSpec in project flink by apache.

the class DynamicTableSourceSpecSerdeTest method testDynamicTableSinkSpecSerde.

public static Stream<DynamicTableSourceSpec> testDynamicTableSinkSpecSerde() {
    Map<String, String> options1 = new HashMap<>();
    options1.put("connector", FileSystemTableFactory.IDENTIFIER);
    options1.put("format", TestCsvFormatFactory.IDENTIFIER);
    options1.put("path", "/tmp");
    final ResolvedSchema resolvedSchema1 = new ResolvedSchema(Collections.singletonList(Column.physical("a", DataTypes.BIGINT())), Collections.emptyList(), null);
    final CatalogTable catalogTable1 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema1).build(), null, Collections.emptyList(), options1);
    DynamicTableSourceSpec spec1 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable1, resolvedSchema1)), null);
    Map<String, String> options2 = new HashMap<>();
    options2.put("connector", TestValuesTableFactory.IDENTIFIER);
    options2.put("disable-lookup", "true");
    options2.put("enable-watermark-push-down", "true");
    options2.put("filterable-fields", "b");
    options2.put("bounded", "false");
    options2.put("readable-metadata", "m1:INT, m2:STRING");
    final ResolvedSchema resolvedSchema2 = new ResolvedSchema(Arrays.asList(Column.physical("a", DataTypes.BIGINT()), Column.physical("b", DataTypes.INT()), Column.physical("c", DataTypes.STRING()), Column.physical("p", DataTypes.STRING()), Column.metadata("m1", DataTypes.INT(), null, false), Column.metadata("m2", DataTypes.STRING(), null, false), Column.physical("ts", DataTypes.TIMESTAMP(3))), Collections.emptyList(), null);
    final CatalogTable catalogTable2 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema2).build(), null, Collections.emptyList(), options2);
    FlinkTypeFactory factory = FlinkTypeFactory.INSTANCE();
    RexBuilder rexBuilder = new RexBuilder(factory);
    DynamicTableSourceSpec spec2 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable2, resolvedSchema2)), Arrays.asList(new ProjectPushDownSpec(new int[][] { { 0 }, { 1 }, { 4 }, { 6 } }, RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new ReadingMetadataSpec(Arrays.asList("m1", "m2"), RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new FilterPushDownSpec(Collections.singletonList(// b >= 10
    rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.INTEGER), 1), rexBuilder.makeExactLiteral(new BigDecimal(10))))), new WatermarkPushDownSpec(rexBuilder.makeCall(SqlStdOperatorTable.MINUS, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.TIMESTAMP, 3), 3), rexBuilder.makeIntervalLiteral(BigDecimal.valueOf(1000), new SqlIntervalQualifier(TimeUnit.SECOND, 2, TimeUnit.SECOND, 6, SqlParserPos.ZERO))), 5000, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new SourceWatermarkSpec(true, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new LimitPushDownSpec(100), new PartitionPushDownSpec(Arrays.asList(new HashMap<String, String>() {

        {
            put("p", "A");
        }
    }, new HashMap<String, String>() {

        {
            put("p", "B");
        }
    }))));
    return Stream.of(spec1, spec2);
}

Example 4 with LimitPushDownSpec

use of org.apache.flink.table.planner.plan.abilities.source.LimitPushDownSpec in project flink by apache.

the class PushLimitIntoTableSourceScanRule method matches.

@Override
public boolean matches(RelOptRuleCall call) {
    Sort sort = call.rel(0);
    TableSourceTable tableSourceTable = call.rel(1).getTable().unwrap(TableSourceTable.class);
    // a limit can be pushed down only if it satisfies the two conditions: 1) do not have order
    // by keys, 2) have limit.
    boolean onlyLimit = sort.getCollation().getFieldCollations().isEmpty() && sort.fetch != null;
    return onlyLimit && tableSourceTable != null && tableSourceTable.tableSource() instanceof SupportsLimitPushDown && Arrays.stream(tableSourceTable.abilitySpecs()).noneMatch(spec -> spec instanceof LimitPushDownSpec);
}