Examples with DynamicTableSink org.apache.flink.table.connector.sink.DynamicTableSink used on opensource projects

Example 31 with DynamicTableSink

use of org.apache.flink.table.connector.sink.DynamicTableSink in project flink by apache.

the class JdbcDynamicTableFactoryTest method testJdbcSinkProperties.

@Test
public void testJdbcSinkProperties() {
    Map<String, String> properties = getAllOptions();
    properties.put("sink.buffer-flush.max-rows", "1000");
    properties.put("sink.buffer-flush.interval", "2min");
    properties.put("sink.max-retries", "5");
    DynamicTableSink actual = createTableSink(SCHEMA, properties);
    JdbcConnectorOptions options = JdbcConnectorOptions.builder().setDBUrl("jdbc:derby:memory:mydb").setTableName("mytable").build();
    JdbcExecutionOptions executionOptions = JdbcExecutionOptions.builder().withBatchSize(1000).withBatchIntervalMs(120_000).withMaxRetries(5).build();
    JdbcDmlOptions dmlOptions = JdbcDmlOptions.builder().withTableName(options.getTableName()).withDialect(options.getDialect()).withFieldNames(SCHEMA.getColumnNames().toArray(new String[0])).withKeyFields("bbb", "aaa").build();
    JdbcDynamicTableSink expected = new JdbcDynamicTableSink(options, executionOptions, dmlOptions, SCHEMA.toPhysicalRowDataType());
    assertEquals(expected, actual);
}

Example 32 with DynamicTableSink

use of org.apache.flink.table.connector.sink.DynamicTableSink in project flink by apache.

the class DebeziumJsonFormatFactoryTest method testSeDeSchema.

@Test
public void testSeDeSchema() {
    final DebeziumJsonDeserializationSchema expectedDeser = new DebeziumJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), InternalTypeInfo.of(PHYSICAL_TYPE), false, true, TimestampFormat.ISO_8601);
    final Map<String, String> options = getAllOptions();
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, PHYSICAL_DATA_TYPE);
    assertEquals(expectedDeser, actualDeser);
    final DebeziumJsonSerializationSchema expectedSer = new DebeziumJsonSerializationSchema((RowType) PHYSICAL_DATA_TYPE.getLogicalType(), TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    final DynamicTableSink actualSink = createTableSink(SCHEMA, options);
    assert actualSink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
    TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
    SerializationSchema<RowData> actualSer = sinkMock.valueFormat.createRuntimeEncoder(new SinkRuntimeProviderContext(false), PHYSICAL_DATA_TYPE);
    assertEquals(expectedSer, actualSer);
}

Example 33 with DynamicTableSink

use of org.apache.flink.table.connector.sink.DynamicTableSink in project flink by apache.

the class DebeziumJsonFormatFactoryTest method testSchemaIncludeOption.

@Test
public void testSchemaIncludeOption() {
    Map<String, String> options = getAllOptions();
    options.put("debezium-json.schema-include", "true");
    final DebeziumJsonDeserializationSchema expectedDeser = new DebeziumJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), InternalTypeInfo.of(PHYSICAL_DATA_TYPE.getLogicalType()), true, true, TimestampFormat.ISO_8601);
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, PHYSICAL_DATA_TYPE);
    assertEquals(expectedDeser, actualDeser);
    try {
        final DynamicTableSink actualSink = createTableSink(SCHEMA, options);
        TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
        // should fail
        sinkMock.valueFormat.createRuntimeEncoder(new SinkRuntimeProviderContext(false), PHYSICAL_DATA_TYPE);
        fail();
    } catch (Exception e) {
        assertEquals(e.getCause().getCause().getMessage(), "Debezium JSON serialization doesn't support " + "'debezium-json.schema-include' option been set to true.");
    }
}

Example 34 with DynamicTableSink

use of org.apache.flink.table.connector.sink.DynamicTableSink in project flink by apache.

the class CanalJsonFormatFactoryTest method createSerializationSchema.

private static SerializationSchema<RowData> createSerializationSchema(Map<String, String> options) {
    DynamicTableSink sink = createTableSink(SCHEMA, options);
    assert sink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
    TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) sink;
    return sinkMock.valueFormat.createRuntimeEncoder(new SinkRuntimeProviderContext(false), PHYSICAL_DATA_TYPE);
}

Example 35 with DynamicTableSink

use of org.apache.flink.table.connector.sink.DynamicTableSink in project flink by apache.

the class MaxwellJsonFormatFactoryTest method testSeDeSchema.

@Test
public void testSeDeSchema() {
    final MaxwellJsonDeserializationSchema expectedDeser = new MaxwellJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), ROW_TYPE_INFO, true, TimestampFormat.ISO_8601);
    final MaxwellJsonSerializationSchema expectedSer = new MaxwellJsonSerializationSchema(PHYSICAL_TYPE, TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    final Map<String, String> options = getAllOptions();
    final DynamicTableSource actualSource = createTableSource(SCHEMA, options);
    assert actualSource instanceof TestDynamicTableFactory.DynamicTableSourceMock;
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedDeser, actualDeser);
    final DynamicTableSink actualSink = createTableSink(SCHEMA, options);
    assert actualSink instanceof TestDynamicTableFactory.DynamicTableSinkMock;
    TestDynamicTableFactory.DynamicTableSinkMock sinkMock = (TestDynamicTableFactory.DynamicTableSinkMock) actualSink;
    SerializationSchema<RowData> actualSer = sinkMock.valueFormat.createRuntimeEncoder(new SinkRuntimeProviderContext(false), SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedSer, actualSer);
}