Examples with DynamicTableSource org.apache.flink.table.connector.source.DynamicTableSource used on opensource projects

Example 31 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class PushFilterIntoSourceScanRuleBase method resolveFiltersAndCreateTableSourceTable.

/**
 * Resolves filters using the underlying sources {@link SupportsFilterPushDown} and creates a
 * new {@link TableSourceTable} with the supplied predicates.
 *
 * @param convertiblePredicates Predicates to resolve
 * @param oldTableSourceTable TableSourceTable to copy
 * @param scan Underlying table scan to push to
 * @param relBuilder Builder to push the scan to
 * @return A tuple, constituting of the resolved filters and the newly created {@link
 *     TableSourceTable}
 */
protected Tuple2<SupportsFilterPushDown.Result, TableSourceTable> resolveFiltersAndCreateTableSourceTable(RexNode[] convertiblePredicates, TableSourceTable oldTableSourceTable, TableScan scan, RelBuilder relBuilder) {
    // record size before applyFilters for update statistics
    int originPredicatesSize = convertiblePredicates.length;
    // update DynamicTableSource
    DynamicTableSource newTableSource = oldTableSourceTable.tableSource().copy();
    SupportsFilterPushDown.Result result = FilterPushDownSpec.apply(Arrays.asList(convertiblePredicates), newTableSource, SourceAbilityContext.from(scan));
    relBuilder.push(scan);
    List<RexNode> acceptedPredicates = convertExpressionToRexNode(result.getAcceptedFilters(), relBuilder);
    FilterPushDownSpec filterPushDownSpec = new FilterPushDownSpec(acceptedPredicates);
    // record size after applyFilters for update statistics
    int updatedPredicatesSize = result.getRemainingFilters().size();
    // set the newStatistic newTableSource and sourceAbilitySpecs
    TableSourceTable newTableSourceTable = oldTableSourceTable.copy(newTableSource, getNewFlinkStatistic(oldTableSourceTable, originPredicatesSize, updatedPredicatesSize), new SourceAbilitySpec[] { filterPushDownSpec });
    return new Tuple2<>(result, newTableSourceTable);
}

Example 32 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class PushLimitIntoTableSourceScanRule method applyLimit.

private TableSourceTable applyLimit(long limit, FlinkLogicalTableSourceScan scan) {
    TableSourceTable relOptTable = scan.getTable().unwrap(TableSourceTable.class);
    TableSourceTable oldTableSourceTable = relOptTable.unwrap(TableSourceTable.class);
    DynamicTableSource newTableSource = oldTableSourceTable.tableSource().copy();
    LimitPushDownSpec limitPushDownSpec = new LimitPushDownSpec(limit);
    limitPushDownSpec.apply(newTableSource, SourceAbilityContext.from(scan));
    FlinkStatistic statistic = relOptTable.getStatistic();
    final long newRowCount;
    if (statistic.getRowCount() != null) {
        newRowCount = Math.min(limit, statistic.getRowCount().longValue());
    } else {
        newRowCount = limit;
    }
    // update TableStats after limit push down
    TableStats newTableStats = new TableStats(newRowCount);
    FlinkStatistic newStatistic = FlinkStatistic.builder().statistic(statistic).tableStats(newTableStats).build();
    return oldTableSourceTable.copy(newTableSource, newStatistic, new SourceAbilitySpec[] { limitPushDownSpec });
}

Example 33 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class RawFormatFactoryTest method createDeserializationSchema.

// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
private static DeserializationSchema<RowData> createDeserializationSchema(ResolvedSchema schema, Map<String, String> options) {
    final DynamicTableSource actualSource = createTableSource(schema, options);
    assertThat(actualSource, instanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class));
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    return scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, schema.toPhysicalRowDataType());
}

Example 34 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class FileSystemTableFactoryTest method testSupportsMetadata.

@Test
public void testSupportsMetadata() {
    Map<String, String> descriptor = new HashMap<>();
    descriptor.put(FactoryUtil.CONNECTOR.key(), "filesystem");
    descriptor.put("path", "/tmp");
    descriptor.put("format", "testcsv");
    descriptor.put("testcsv.my_option", "my_value");
    DynamicTableSource source = createTableSource(SCHEMA, descriptor);
    assertTrue(source instanceof FileSystemTableSource);
    Map<String, DataType> readableMetadata = new HashMap<>();
    readableMetadata.put("file.path", DataTypes.STRING().notNull());
    readableMetadata.put("file.name", DataTypes.STRING().notNull());
    readableMetadata.put("file.size", DataTypes.BIGINT().notNull());
    readableMetadata.put("file.modification-time", DataTypes.TIMESTAMP_LTZ(3).notNull());
    assertEquals(readableMetadata, ((FileSystemTableSource) source).listReadableMetadata());
}

Example 35 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class AvroFormatFactoryTest method testSeDeSchema.

@Test
public void testSeDeSchema() {
    final AvroRowDataDeserializationSchema expectedDeser = new AvroRowDataDeserializationSchema(ROW_TYPE, InternalTypeInfo.of(ROW_TYPE));
    final Map<String, String> options = getAllOptions();
    final DynamicTableSource actualSource = FactoryMocks.createTableSource(SCHEMA, options);
    assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedDeser, actualDeser);
    final AvroRowDataSerializationSchema expectedSer = new AvroRowDataSerializationSchema(ROW_TYPE);
    final DynamicTableSink actualSink = FactoryMocks.createTableSink(SCHEMA, options);
    assert actualSink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
    TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
    SerializationSchema<RowData> actualSer = sinkMock.valueFormat.createRuntimeEncoder(null, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedSer, actualSer);
}