Examples with DynamicTableSource org.apache.flink.table.connector.source.DynamicTableSource used on opensource projects

Example 36 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class DebeziumAvroFormatFactoryTest method createDeserializationSchema.

private static DeserializationSchema<RowData> createDeserializationSchema(Map<String, String> options) {
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    assertThat(actualSource, instanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class));
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    return scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType());
}

Example 37 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class JdbcDynamicTableFactoryTest method testJdbcReadProperties.

@Test
public void testJdbcReadProperties() {
    Map<String, String> properties = getAllOptions();
    properties.put("scan.partition.column", "aaa");
    properties.put("scan.partition.lower-bound", "-10");
    properties.put("scan.partition.upper-bound", "100");
    properties.put("scan.partition.num", "10");
    properties.put("scan.fetch-size", "20");
    properties.put("scan.auto-commit", "false");
    DynamicTableSource actual = createTableSource(SCHEMA, properties);
    JdbcConnectorOptions options = JdbcConnectorOptions.builder().setDBUrl("jdbc:derby:memory:mydb").setTableName("mytable").build();
    JdbcReadOptions readOptions = JdbcReadOptions.builder().setPartitionColumnName("aaa").setPartitionLowerBound(-10).setPartitionUpperBound(100).setNumPartitions(10).setFetchSize(20).setAutoCommit(false).build();
    JdbcLookupOptions lookupOptions = JdbcLookupOptions.builder().setCacheMaxSize(-1).setCacheExpireMs(10_000).setMaxRetryTimes(3).build();
    JdbcDynamicTableSource expected = new JdbcDynamicTableSource(options, readOptions, lookupOptions, SCHEMA.toPhysicalRowDataType());
    assertEquals(expected, actual);
}

Example 38 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class DebeziumJsonFormatFactoryTest method testSeDeSchema.

@Test
public void testSeDeSchema() {
    final DebeziumJsonDeserializationSchema expectedDeser = new DebeziumJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), InternalTypeInfo.of(PHYSICAL_TYPE), false, true, TimestampFormat.ISO_8601);
    final Map<String, String> options = getAllOptions();
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, PHYSICAL_DATA_TYPE);
    assertEquals(expectedDeser, actualDeser);
    final DebeziumJsonSerializationSchema expectedSer = new DebeziumJsonSerializationSchema((RowType) PHYSICAL_DATA_TYPE.getLogicalType(), TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    final DynamicTableSink actualSink = createTableSink(SCHEMA, options);
    assert actualSink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
    TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
    SerializationSchema<RowData> actualSer = sinkMock.valueFormat.createRuntimeEncoder(new SinkRuntimeProviderContext(false), PHYSICAL_DATA_TYPE);
    assertEquals(expectedSer, actualSer);
}

Example 39 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class DebeziumJsonFormatFactoryTest method testSchemaIncludeOption.

@Test
public void testSchemaIncludeOption() {
    Map<String, String> options = getAllOptions();
    options.put("debezium-json.schema-include", "true");
    final DebeziumJsonDeserializationSchema expectedDeser = new DebeziumJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), InternalTypeInfo.of(PHYSICAL_DATA_TYPE.getLogicalType()), true, true, TimestampFormat.ISO_8601);
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, PHYSICAL_DATA_TYPE);
    assertEquals(expectedDeser, actualDeser);
    try {
        final DynamicTableSink actualSink = createTableSink(SCHEMA, options);
        TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
        // should fail
        sinkMock.valueFormat.createRuntimeEncoder(new SinkRuntimeProviderContext(false), PHYSICAL_DATA_TYPE);
        fail();
    } catch (Exception e) {
        assertEquals(e.getCause().getCause().getMessage(), "Debezium JSON serialization doesn't support " + "'debezium-json.schema-include' option been set to true.");
    }
}

Example 40 with DynamicTableSource

use of org.apache.flink.table.connector.source.DynamicTableSource in project flink by apache.

the class MaxwellJsonFormatFactoryTest method testSeDeSchema.

@Test
public void testSeDeSchema() {
    final MaxwellJsonDeserializationSchema expectedDeser = new MaxwellJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), ROW_TYPE_INFO, true, TimestampFormat.ISO_8601);
    final MaxwellJsonSerializationSchema expectedSer = new MaxwellJsonSerializationSchema(PHYSICAL_TYPE, TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    final Map<String, String> options = getAllOptions();
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedDeser, actualDeser);
    final DynamicTableSink actualSink = createTableSink(SCHEMA, options);
    assert actualSink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
    TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
    SerializationSchema<RowData> actualSer = sinkMock.valueFormat.createRuntimeEncoder(new SinkRuntimeProviderContext(false), SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedSer, actualSer);
}