Examples with JdbcLookupOptions org.apache.flink.connector.jdbc.internal.options.JdbcLookupOptions used on opensource projects

Example 1 with JdbcLookupOptions

use of org.apache.flink.connector.jdbc.internal.options.JdbcLookupOptions in project flink by apache.

the class JdbcDynamicTableFactoryTest method testJdbcLookupPropertiesWithExcludeEmptyResult.

@Test
public void testJdbcLookupPropertiesWithExcludeEmptyResult() {
    Map<String, String> properties = getAllOptions();
    properties.put("lookup.cache.max-rows", "1000");
    properties.put("lookup.cache.ttl", "10s");
    properties.put("lookup.max-retries", "10");
    properties.put("lookup.cache.caching-missing-key", "true");
    DynamicTableSource actual = createTableSource(SCHEMA, properties);
    JdbcConnectorOptions options = JdbcConnectorOptions.builder().setDBUrl("jdbc:derby:memory:mydb").setTableName("mytable").build();
    JdbcLookupOptions lookupOptions = JdbcLookupOptions.builder().setCacheMaxSize(1000).setCacheExpireMs(10_000).setMaxRetryTimes(10).setCacheMissingKey(true).build();
    JdbcDynamicTableSource expected = new JdbcDynamicTableSource(options, JdbcReadOptions.builder().build(), lookupOptions, SCHEMA.toPhysicalRowDataType());
    assertEquals(expected, actual);
}

Example 2 with JdbcLookupOptions

use of org.apache.flink.connector.jdbc.internal.options.JdbcLookupOptions in project flink by apache.

the class JdbcDynamicTableFactoryTest method testJdbcLookupProperties.

@Test
public void testJdbcLookupProperties() {
    Map<String, String> properties = getAllOptions();
    properties.put("lookup.cache.max-rows", "1000");
    properties.put("lookup.cache.ttl", "10s");
    properties.put("lookup.max-retries", "10");
    DynamicTableSource actual = createTableSource(SCHEMA, properties);
    JdbcConnectorOptions options = JdbcConnectorOptions.builder().setDBUrl("jdbc:derby:memory:mydb").setTableName("mytable").build();
    JdbcLookupOptions lookupOptions = JdbcLookupOptions.builder().setCacheMaxSize(1000).setCacheExpireMs(10_000).setMaxRetryTimes(10).build();
    JdbcDynamicTableSource expected = new JdbcDynamicTableSource(options, JdbcReadOptions.builder().build(), lookupOptions, SCHEMA.toPhysicalRowDataType());
    assertEquals(expected, actual);
}

Example 3 with JdbcLookupOptions

use of org.apache.flink.connector.jdbc.internal.options.JdbcLookupOptions in project flink by apache.

the class JdbcDynamicTableFactoryTest method testJdbcCommonProperties.

@Test
public void testJdbcCommonProperties() {
    Map<String, String> properties = getAllOptions();
    properties.put("driver", "org.apache.derby.jdbc.EmbeddedDriver");
    properties.put("username", "user");
    properties.put("password", "pass");
    properties.put("connection.max-retry-timeout", "120s");
    // validation for source
    DynamicTableSource actualSource = createTableSource(SCHEMA, properties);
    JdbcConnectorOptions options = JdbcConnectorOptions.builder().setDBUrl("jdbc:derby:memory:mydb").setTableName("mytable").setDriverName("org.apache.derby.jdbc.EmbeddedDriver").setUsername("user").setPassword("pass").setConnectionCheckTimeoutSeconds(120).build();
    JdbcLookupOptions lookupOptions = JdbcLookupOptions.builder().setCacheMaxSize(-1).setCacheExpireMs(10_000).setMaxRetryTimes(3).build();
    JdbcDynamicTableSource expectedSource = new JdbcDynamicTableSource(options, JdbcReadOptions.builder().build(), lookupOptions, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedSource, actualSource);
    // validation for sink
    DynamicTableSink actualSink = createTableSink(SCHEMA, properties);
    // default flush configurations
    JdbcExecutionOptions executionOptions = JdbcExecutionOptions.builder().withBatchSize(100).withBatchIntervalMs(1000).withMaxRetries(3).build();
    JdbcDmlOptions dmlOptions = JdbcDmlOptions.builder().withTableName(options.getTableName()).withDialect(options.getDialect()).withFieldNames(SCHEMA.getColumnNames().toArray(new String[0])).withKeyFields("bbb", "aaa").build();
    JdbcDynamicTableSink expectedSink = new JdbcDynamicTableSink(options, executionOptions, dmlOptions, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedSink, actualSink);
}

Example 4 with JdbcLookupOptions

use of org.apache.flink.connector.jdbc.internal.options.JdbcLookupOptions in project flink by apache.

the class JdbcRowDataLookupFunctionTest method testEvalWithCacheMissingKeyPositive.

@Test
public void testEvalWithCacheMissingKeyPositive() throws Exception {
    JdbcLookupOptions lookupOptions = JdbcLookupOptions.builder().setCacheMissingKey(true).setCacheExpireMs(60000).setCacheMaxSize(10).build();
    JdbcRowDataLookupFunction lookupFunction = buildRowDataLookupFunction(lookupOptions);
    ListOutputCollector collector = new ListOutputCollector();
    lookupFunction.setCollector(collector);
    lookupFunction.open(null);
    lookupFunction.eval(4, StringData.fromString("9"));
    RowData keyRow = GenericRowData.of(4, StringData.fromString("9"));
    Cache<RowData, List<RowData>> cache = lookupFunction.getCache();
    // empty data should cache
    assertEquals(cache.getIfPresent(keyRow), Collections.<RowData>emptyList());
    // put db entry for keyRow
    // final cache output should also be empty till TTL expires
    insert("INSERT INTO " + LOOKUP_TABLE + " (id1, id2, comment1, comment2) VALUES (4, '9', '49-c1', '49-c2')");
    lookupFunction.eval(4, StringData.fromString("9"));
    assertEquals(cache.getIfPresent(keyRow), Collections.<RowData>emptyList());
}

Example 5 with JdbcLookupOptions

use of org.apache.flink.connector.jdbc.internal.options.JdbcLookupOptions in project flink by apache.

the class JdbcRowDataLookupFunctionTest method testEvalWithCacheMissingKeyNegative.

@Test
public void testEvalWithCacheMissingKeyNegative() throws Exception {
    JdbcLookupOptions lookupOptions = JdbcLookupOptions.builder().setCacheMissingKey(false).setCacheExpireMs(60000).setCacheMaxSize(10).build();
    JdbcRowDataLookupFunction lookupFunction = buildRowDataLookupFunction(lookupOptions);
    ListOutputCollector collector = new ListOutputCollector();
    lookupFunction.setCollector(collector);
    lookupFunction.open(null);
    lookupFunction.eval(5, StringData.fromString("1"));
    RowData keyRow = GenericRowData.of(5, StringData.fromString("1"));
    Cache<RowData, List<RowData>> cache = lookupFunction.getCache();
    // empty data should not get cached
    assert cache.getIfPresent(keyRow) == null;
    // put db entry for keyRow
    // final cache output should contain data
    insert("INSERT INTO " + LOOKUP_TABLE + " (id1, id2, comment1, comment2) VALUES (5, '1', '51-c1', '51-c2')");
    lookupFunction.eval(5, StringData.fromString("1"));
    List<RowData> expectedOutput = new ArrayList<>();
    expectedOutput.add(GenericRowData.of(5, StringData.fromString("1"), StringData.fromString("51-c1"), StringData.fromString("51-c2")));
    assertEquals(cache.getIfPresent(keyRow), expectedOutput);
}