Examples with JdbcSplit io.prestosql.plugin.jdbc.JdbcSplit used on opensource projects

Example 6 with JdbcSplit

use of io.prestosql.plugin.jdbc.JdbcSplit in project hetu-core by openlookeng.

the class TestDataSourceTableSplitManager method testGetTableSplits12.

@Test
public void testGetTableSplits12() {
    long[][] rangeArray = new long[][] { { 0, 2 }, { 2, 4 }, { 4, 5 } };
    BaseJdbcConfig config = new BaseJdbcConfig();
    config.setPushDownEnable(true).setTableSplitEnable(true).setTableSplitFields("[{\"catalogName\":\"" + catalogName + "\",\"schemaName\":\"EXAMPLE\",\"tableName\":\"NUMBERS\"," + "\"splitField\":\"value\",\"calcStepEnable\":\"false\",\"dataReadOnly\":\"true\",\"splitCount\":\"3\"," + "\"fieldMinValue\":\"1\",\"fieldMaxValue\":\"5\"}]");
    tableHandle = getTableHandle(new SchemaTableName("example", "numbers"));
    splitManager = new DataSourceTableSplitManager(config, jdbcClient, nodeManager);
    TableSplitConfig tableSplitConfig = splitManager.getTableSplitConfig(tableHandle);
    tableSplitConfig.setTableSplitFieldValid(true);
    ConnectorSplitSource splitSource = splitManager.getTableSplits(JdbcIdentity.from(SESSION), tableHandle);
    List<ConnectorSplit> splits = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, 1000)).getSplits();
    int index = 0;
    for (ConnectorSplit split : splits) {
        JdbcSplit jdbcSplit = (JdbcSplit) split;
        assertEquals(Long.parseLong(jdbcSplit.getRangeStart()), rangeArray[index][0]);
        assertEquals(Long.parseLong(jdbcSplit.getRangEnd()), rangeArray[index][1]);
        index++;
        assertFalse(index > 3);
    }
}

Example 7 with JdbcSplit

use of io.prestosql.plugin.jdbc.JdbcSplit in project hetu-core by openlookeng.

the class TestDataSourceTableSplitManager method testGetTableSplits02.

// fieldMinValue and fieldMaxValue is null and dataReadOnly is true
@Test
public void testGetTableSplits02() {
    long[][] rangeArray = new long[][] { { 0, 6 }, { 6, 12 } };
    BaseJdbcConfig config = new BaseJdbcConfig();
    config.setPushDownEnable(true).setTableSplitEnable(true).setTableSplitFields("[{\"catalogName\":\"" + catalogName + "\",\"schemaName\":\"EXAMPLE\",\"tableName\":\"NUMBERS\"," + "\"splitField\":\"value\",\"calcStepEnable\":\"false\",\"dataReadOnly\":\"true\",\"splitCount\":\"2\"," + "\"fieldMinValue\":\"\",\"fieldMaxValue\":\"\"}]");
    tableHandle = getTableHandle(new SchemaTableName("example", "numbers"));
    splitManager = new DataSourceTableSplitManager(config, jdbcClient, nodeManager);
    TableSplitConfig tableSplitConfig = splitManager.getTableSplitConfig(tableHandle);
    tableSplitConfig.setTableSplitFieldValid(true);
    ConnectorSplitSource splitSource = splitManager.getTableSplits(JdbcIdentity.from(SESSION), tableHandle);
    List<ConnectorSplit> splits = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, 1000)).getSplits();
    int index = 0;
    for (ConnectorSplit split : splits) {
        JdbcSplit jdbcSplit = (JdbcSplit) split;
        assertEquals(Long.parseLong(jdbcSplit.getRangeStart()), rangeArray[index][0]);
        assertEquals(Long.parseLong(jdbcSplit.getRangEnd()), rangeArray[index][1]);
        index++;
        assertFalse(index > 2);
    }
}

Example 8 with JdbcSplit

use of io.prestosql.plugin.jdbc.JdbcSplit in project hetu-core by openlookeng.

the class DataSourceTableSplitManager method getTableSplits.

public ConnectorSplitSource getTableSplits(JdbcIdentity identity, JdbcTableHandle jdbcTableHandle) {
    List<JdbcSplit> jdbcSplitsList = new ArrayList<>();
    TableSplitConfig splitConfig = tableSplitsMap.get(generateTableFullName(jdbcTableHandle.getCatalogName(), jdbcTableHandle.getSchemaName(), jdbcTableHandle.getTableName()));
    if (splitConfig == null) {
        return getFixedSplitSource(jdbcTableHandle);
    }
    if (splitConfig.getSplitCount() == null || isNullOrEmpty(splitConfig.getSplitField()) || splitConfig.getSplitCount() <= 1) {
        return getFixedSplitSource(jdbcTableHandle);
    }
    // config with wrong split field type
    if (!splitConfig.isTableSplitFieldValid()) {
        log.warn("Table(%s) split field(%s) NOT integer like value type.", generateTableFullName(jdbcTableHandle.getCatalogName(), jdbcTableHandle.getSchemaName(), jdbcTableHandle.getTableName()), splitConfig.getSplitField());
        return getFixedSplitSource(jdbcTableHandle);
    }
    long timeStamp = System.nanoTime();
    if (splitConfig.isCalcStepEnable()) {
        List<SplitStatLog> splitLogs;
        try {
            splitLogs = stepCalcManager.getAdjustSplitList(jdbcTableHandle.getCatalogName(), jdbcTableHandle.getSchemaName(), jdbcTableHandle.getTableName());
            if (null == splitLogs) {
                splitLogs = new ArrayList<SplitStatLog>();
            }
        } catch (PrestoException e) {
            log.error("Get table split from dynamic step calc manager failed, error info:" + e.getMessage());
            return getFixedSplitSource(jdbcTableHandle);
        }
        for (SplitStatLog splitLog : splitLogs) {
            String splitPart = splitConfig.getSplitField() + " > " + splitLog.getBeginIndex() + " and " + splitConfig.getSplitField() + " <= " + splitLog.getEndIndex();
            JdbcSplit jdbcSplit = new JdbcSplit(splitLog.getCatalogName(), splitLog.getSchemaName(), splitLog.getTableName(), splitConfig.getSplitField(), splitLog.getBeginIndex().toString(), splitLog.getEndIndex().toString(), timeStamp, splitConfig.getSplitCount(), Optional.of(splitPart));
            jdbcSplitsList.add(jdbcSplit);
        }
    }
    if ((jdbcSplitsList == null) || (jdbcSplitsList.isEmpty())) {
        Long[] fieldMinAndMaxValue = new Long[2];
        try (Connection connection = jdbcClient.getConnection(identity, (JdbcSplit) null)) {
            fieldMinAndMaxValue = jdbcClient.getSplitFieldMinAndMaxValue(splitConfig, connection, jdbcTableHandle);
            if (fieldMinAndMaxValue == null || fieldMinAndMaxValue[0].equals(fieldMinAndMaxValue[1])) {
                return getFixedSplitSource(jdbcTableHandle);
            }
        } catch (SQLException e) {
            return getFixedSplitSource(jdbcTableHandle);
        }
        splitTable(fieldMinAndMaxValue, jdbcTableHandle, jdbcSplitsList, splitConfig, timeStamp);
    }
    // submit static log, and filter fixed split source
    if (splitConfig.isCalcStepEnable() && (jdbcSplitsList.size() > 1)) {
        Thread collectStaticLogs = new Thread(new Runnable() {

            @Override
            public void run() {
                handleSplitStatic(identity, jdbcSplitsList);
            }
        });
        collectStaticLogs.setName("splitStatic");
        collectStaticLogs.setUncaughtExceptionHandler((tr, ex) -> System.out.println(tr.getName() + ":" + ex.getMessage()));
        collectStaticLogs.start();
    }
    return new DataSourceSplitSource(jdbcSplitsList);
}