Examples with DynamicTableSourceSpec org.apache.flink.table.planner.plan.nodes.exec.spec.DynamicTableSourceSpec used on opensource projects

Example 1 with DynamicTableSourceSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.DynamicTableSourceSpec in project flink by apache.

the class DynamicTableSourceSpecSerdeTest method testDynamicTableSourceSpecSerde.

@ParameterizedTest
@MethodSource("testDynamicTableSinkSpecSerde")
void testDynamicTableSourceSpecSerde(DynamicTableSourceSpec spec) throws IOException {
    PlannerMocks plannerMocks = PlannerMocks.create();
    CatalogManager catalogManager = plannerMocks.getCatalogManager();
    catalogManager.createTable(spec.getContextResolvedTable().getResolvedTable(), spec.getContextResolvedTable().getIdentifier(), false);
    SerdeContext serdeCtx = configuredSerdeContext(catalogManager, plannerMocks.getTableConfig());
    // Re-init the spec to be permanent with correct catalog
    spec = new DynamicTableSourceSpec(ContextResolvedTable.permanent(spec.getContextResolvedTable().getIdentifier(), catalogManager.getCatalog(catalogManager.getCurrentCatalog()).get(), spec.getContextResolvedTable().getResolvedTable()), spec.getSourceAbilities());
    String actualJson = toJson(serdeCtx, spec);
    DynamicTableSourceSpec actual = toObject(serdeCtx, actualJson, DynamicTableSourceSpec.class);
    assertThat(actual.getContextResolvedTable()).isEqualTo(spec.getContextResolvedTable());
    assertThat(actual.getSourceAbilities()).isEqualTo(spec.getSourceAbilities());
    assertThat(actual.getScanTableSource(plannerMocks.getPlannerContext().getFlinkContext())).isNotNull();
}

Example 2 with DynamicTableSourceSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.DynamicTableSourceSpec in project flink by apache.

the class DynamicTableSourceSpecSerdeTest method testDynamicTableSinkSpecSerde.

public static Stream<DynamicTableSourceSpec> testDynamicTableSinkSpecSerde() {
    Map<String, String> options1 = new HashMap<>();
    options1.put("connector", FileSystemTableFactory.IDENTIFIER);
    options1.put("format", TestCsvFormatFactory.IDENTIFIER);
    options1.put("path", "/tmp");
    final ResolvedSchema resolvedSchema1 = new ResolvedSchema(Collections.singletonList(Column.physical("a", DataTypes.BIGINT())), Collections.emptyList(), null);
    final CatalogTable catalogTable1 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema1).build(), null, Collections.emptyList(), options1);
    DynamicTableSourceSpec spec1 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable1, resolvedSchema1)), null);
    Map<String, String> options2 = new HashMap<>();
    options2.put("connector", TestValuesTableFactory.IDENTIFIER);
    options2.put("disable-lookup", "true");
    options2.put("enable-watermark-push-down", "true");
    options2.put("filterable-fields", "b");
    options2.put("bounded", "false");
    options2.put("readable-metadata", "m1:INT, m2:STRING");
    final ResolvedSchema resolvedSchema2 = new ResolvedSchema(Arrays.asList(Column.physical("a", DataTypes.BIGINT()), Column.physical("b", DataTypes.INT()), Column.physical("c", DataTypes.STRING()), Column.physical("p", DataTypes.STRING()), Column.metadata("m1", DataTypes.INT(), null, false), Column.metadata("m2", DataTypes.STRING(), null, false), Column.physical("ts", DataTypes.TIMESTAMP(3))), Collections.emptyList(), null);
    final CatalogTable catalogTable2 = CatalogTable.of(Schema.newBuilder().fromResolvedSchema(resolvedSchema2).build(), null, Collections.emptyList(), options2);
    FlinkTypeFactory factory = FlinkTypeFactory.INSTANCE();
    RexBuilder rexBuilder = new RexBuilder(factory);
    DynamicTableSourceSpec spec2 = new DynamicTableSourceSpec(ContextResolvedTable.temporary(ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "MyTable"), new ResolvedCatalogTable(catalogTable2, resolvedSchema2)), Arrays.asList(new ProjectPushDownSpec(new int[][] { { 0 }, { 1 }, { 4 }, { 6 } }, RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new ReadingMetadataSpec(Arrays.asList("m1", "m2"), RowType.of(new LogicalType[] { new BigIntType(), new IntType(), new IntType(), new TimestampType(3) }, new String[] { "a", "b", "m1", "ts" })), new FilterPushDownSpec(Collections.singletonList(// b >= 10
    rexBuilder.makeCall(SqlStdOperatorTable.GREATER_THAN_OR_EQUAL, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.INTEGER), 1), rexBuilder.makeExactLiteral(new BigDecimal(10))))), new WatermarkPushDownSpec(rexBuilder.makeCall(SqlStdOperatorTable.MINUS, rexBuilder.makeInputRef(factory.createSqlType(SqlTypeName.TIMESTAMP, 3), 3), rexBuilder.makeIntervalLiteral(BigDecimal.valueOf(1000), new SqlIntervalQualifier(TimeUnit.SECOND, 2, TimeUnit.SECOND, 6, SqlParserPos.ZERO))), 5000, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new SourceWatermarkSpec(true, RowType.of(new BigIntType(), new IntType(), new IntType(), new TimestampType(false, TimestampKind.ROWTIME, 3))), new LimitPushDownSpec(100), new PartitionPushDownSpec(Arrays.asList(new HashMap<String, String>() {

        {
            put("p", "A");
        }
    }, new HashMap<String, String>() {

        {
            put("p", "B");
        }
    }))));
    return Stream.of(spec1, spec2);
}

Example 3 with DynamicTableSourceSpec

use of org.apache.flink.table.planner.plan.nodes.exec.spec.DynamicTableSourceSpec in project flink by apache.

the class DynamicTableSourceSpecSerdeTest method testDynamicTableSourceSpecSerdeWithEnrichmentOptions.

@Test
void testDynamicTableSourceSpecSerdeWithEnrichmentOptions() throws Exception {
    // Test model
    ObjectIdentifier identifier = ObjectIdentifier.of(DEFAULT_BUILTIN_CATALOG, DEFAULT_BUILTIN_DATABASE, "my_table");
    String formatPrefix = FactoryUtil.getFormatPrefix(FORMAT, TestFormatFactory.IDENTIFIER);
    Map<String, String> planOptions = new HashMap<>();
    planOptions.put(CONNECTOR.key(), TestDynamicTableFactory.IDENTIFIER);
    planOptions.put(TARGET.key(), "abc");
    planOptions.put(PASSWORD.key(), "abc");
    planOptions.put(FORMAT.key(), TestFormatFactory.IDENTIFIER);
    planOptions.put(formatPrefix + DELIMITER.key(), "|");
    Map<String, String> catalogOptions = new HashMap<>();
    catalogOptions.put(CONNECTOR.key(), TestDynamicTableFactory.IDENTIFIER);
    catalogOptions.put(TARGET.key(), "abc");
    catalogOptions.put(PASSWORD.key(), "xyz");
    catalogOptions.put(FORMAT.key(), TestFormatFactory.IDENTIFIER);
    catalogOptions.put(formatPrefix + DELIMITER.key(), ",");
    ResolvedCatalogTable planResolvedCatalogTable = tableWithOnlyPhysicalColumns(planOptions);
    ResolvedCatalogTable catalogResolvedCatalogTable = tableWithOnlyPhysicalColumns(catalogOptions);
    // Create planner mocks
    PlannerMocks plannerMocks = PlannerMocks.create(new Configuration().set(PLAN_RESTORE_CATALOG_OBJECTS, CatalogPlanRestore.ALL).set(PLAN_COMPILE_CATALOG_OBJECTS, CatalogPlanCompilation.ALL));
    CatalogManager catalogManager = plannerMocks.getCatalogManager();
    catalogManager.createTable(catalogResolvedCatalogTable, identifier, false);
    // Mock the context
    SerdeContext serdeCtx = configuredSerdeContext(catalogManager, plannerMocks.getTableConfig());
    DynamicTableSourceSpec planSpec = new DynamicTableSourceSpec(ContextResolvedTable.permanent(identifier, catalogManager.getCatalog(catalogManager.getCurrentCatalog()).get(), planResolvedCatalogTable), Collections.emptyList());
    String actualJson = toJson(serdeCtx, planSpec);
    DynamicTableSourceSpec actual = toObject(serdeCtx, actualJson, DynamicTableSourceSpec.class);
    assertThat(actual.getContextResolvedTable()).isEqualTo(planSpec.getContextResolvedTable());
    assertThat(actual.getSourceAbilities()).isNull();
    TestDynamicTableFactory.DynamicTableSourceMock dynamicTableSource = (TestDynamicTableFactory.DynamicTableSourceMock) actual.getScanTableSource(plannerMocks.getPlannerContext().getFlinkContext());
    assertThat(dynamicTableSource.password).isEqualTo("xyz");
    assertThat(((TestFormatFactory.DecodingFormatMock) dynamicTableSource.valueFormat).delimiter).isEqualTo(",");
}