Examples with ParameterValuesProvider org.apache.flink.api.java.io.jdbc.split.ParameterValuesProvider used on opensource projects

Example 1 with ParameterValuesProvider

use of org.apache.flink.api.java.io.jdbc.split.ParameterValuesProvider in project flink by apache.

the class JDBCInputFormatTest method testJDBCInputFormatWithParallelismAndNumericColumnSplitting.

@Test
public void testJDBCInputFormatWithParallelismAndNumericColumnSplitting() throws IOException, InstantiationException, IllegalAccessException {
    final int fetchSize = 1;
    final Long min = new Long(JDBCTestBase.testData[0][0] + "");
    final Long max = new Long(JDBCTestBase.testData[JDBCTestBase.testData.length - fetchSize][0] + "");
    ParameterValuesProvider pramProvider = new NumericBetweenParametersProvider(fetchSize, min, max);
    jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat().setDrivername(DRIVER_CLASS).setDBUrl(DB_URL).setQuery(JDBCTestBase.SELECT_ALL_BOOKS_SPLIT_BY_ID).setRowTypeInfo(rowTypeInfo).setParametersProvider(pramProvider).setResultSetType(ResultSet.TYPE_SCROLL_INSENSITIVE).finish();
    jdbcInputFormat.openInputFormat();
    InputSplit[] splits = jdbcInputFormat.createInputSplits(1);
    //this query exploit parallelism (1 split for every id)
    Assert.assertEquals(testData.length, splits.length);
    int recordCount = 0;
    Row row = new Row(5);
    for (int i = 0; i < splits.length; i++) {
        jdbcInputFormat.open(splits[i]);
        while (!jdbcInputFormat.reachedEnd()) {
            Row next = jdbcInputFormat.nextRecord(row);
            if (next == null) {
                break;
            }
            if (next.getField(0) != null) {
                Assert.assertEquals("Field 0 should be int", Integer.class, next.getField(0).getClass());
            }
            if (next.getField(1) != null) {
                Assert.assertEquals("Field 1 should be String", String.class, next.getField(1).getClass());
            }
            if (next.getField(2) != null) {
                Assert.assertEquals("Field 2 should be String", String.class, next.getField(2).getClass());
            }
            if (next.getField(3) != null) {
                Assert.assertEquals("Field 3 should be float", Double.class, next.getField(3).getClass());
            }
            if (next.getField(4) != null) {
                Assert.assertEquals("Field 4 should be int", Integer.class, next.getField(4).getClass());
            }
            for (int x = 0; x < 5; x++) {
                if (testData[recordCount][x] != null) {
                    Assert.assertEquals(testData[recordCount][x], next.getField(x));
                }
            }
            recordCount++;
        }
        jdbcInputFormat.close();
    }
    jdbcInputFormat.closeInputFormat();
    Assert.assertEquals(testData.length, recordCount);
}

Example 2 with ParameterValuesProvider

use of org.apache.flink.api.java.io.jdbc.split.ParameterValuesProvider in project flink by apache.

the class JDBCInputFormatTest method testJDBCInputFormatWithParallelismAndGenericSplitting.

@Test
public void testJDBCInputFormatWithParallelismAndGenericSplitting() throws IOException, InstantiationException, IllegalAccessException {
    Serializable[][] queryParameters = new String[2][1];
    queryParameters[0] = new String[] { "Kumar" };
    queryParameters[1] = new String[] { "Tan Ah Teck" };
    ParameterValuesProvider paramProvider = new GenericParameterValuesProvider(queryParameters);
    jdbcInputFormat = JDBCInputFormat.buildJDBCInputFormat().setDrivername(DRIVER_CLASS).setDBUrl(DB_URL).setQuery(JDBCTestBase.SELECT_ALL_BOOKS_SPLIT_BY_AUTHOR).setRowTypeInfo(rowTypeInfo).setParametersProvider(paramProvider).setResultSetType(ResultSet.TYPE_SCROLL_INSENSITIVE).finish();
    jdbcInputFormat.openInputFormat();
    InputSplit[] splits = jdbcInputFormat.createInputSplits(1);
    //this query exploit parallelism (1 split for every queryParameters row)
    Assert.assertEquals(queryParameters.length, splits.length);
    int recordCount = 0;
    Row row = new Row(5);
    for (int i = 0; i < splits.length; i++) {
        jdbcInputFormat.open(splits[i]);
        while (!jdbcInputFormat.reachedEnd()) {
            Row next = jdbcInputFormat.nextRecord(row);
            if (next == null) {
                break;
            }
            if (next.getField(0) != null) {
                Assert.assertEquals("Field 0 should be int", Integer.class, next.getField(0).getClass());
            }
            if (next.getField(1) != null) {
                Assert.assertEquals("Field 1 should be String", String.class, next.getField(1).getClass());
            }
            if (next.getField(2) != null) {
                Assert.assertEquals("Field 2 should be String", String.class, next.getField(2).getClass());
            }
            if (next.getField(3) != null) {
                Assert.assertEquals("Field 3 should be float", Double.class, next.getField(3).getClass());
            }
            if (next.getField(4) != null) {
                Assert.assertEquals("Field 4 should be int", Integer.class, next.getField(4).getClass());
            }
            recordCount++;
        }
        jdbcInputFormat.close();
    }
    Assert.assertEquals(3, recordCount);
    jdbcInputFormat.closeInputFormat();
}