Examples with DbMapping com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping used on opensource projects

Example 6 with DbMapping

use of com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping in project canal by alibaba.

the class PhoenixEtlService method importData.

/**
 * 导入数据
 */
public static EtlResult importData(DataSource srcDS, Connection targetDSConnection, MappingConfig config, List<String> params) {
    EtlResult etlResult = new EtlResult();
    AtomicLong successCount = new AtomicLong();
    List<String> errMsg = new ArrayList<>();
    String hbaseTable = "";
    try {
        if (config == null) {
            logger.error("Config is null!");
            etlResult.setSucceeded(false);
            etlResult.setErrorMessage("Config is null!");
            return etlResult;
        }
        boolean debug = params != null && params.get(0).equals("_debug");
        if (debug) {
            params = params.subList(1, params.size());
        }
        syncSchema(srcDS, targetDSConnection, config);
        DbMapping dbMapping = config.getDbMapping();
        long start = System.currentTimeMillis();
        // 拼接sql
        StringBuilder sql = new StringBuilder("SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable());
        // 拼接条件
        appendCondition(params, dbMapping, srcDS, sql);
        // 获取总数
        String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
        long cnt = (Long) Util.sqlRS(srcDS, countSql, rs -> {
            Long count = null;
            try {
                if (rs.next()) {
                    count = ((Number) rs.getObject(1)).longValue();
                }
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
            }
            return count == null ? 0 : count;
        });
        // 当大于1万条记录时开启多线程
        if (cnt >= 10000) {
            int threadCount = 3;
            long perThreadCnt = cnt / threadCount;
            ExecutorService executor = Util.newFixedThreadPool(threadCount, 5000L);
            for (int i = 0; i < threadCount; i++) {
                long offset = i * perThreadCnt;
                Long size = null;
                if (i != threadCount - 1) {
                    size = perThreadCnt;
                }
                String sqlFinal;
                if (size != null) {
                    sqlFinal = sql + " LIMIT " + offset + "," + size;
                } else {
                    sqlFinal = sql + " LIMIT " + offset + "," + cnt;
                }
                executor.execute(() -> executeSqlImport(srcDS, targetDSConnection, sqlFinal, dbMapping, successCount, errMsg, debug));
            }
            executor.shutdown();
            // noinspection StatementWithEmptyBody
            while (!executor.awaitTermination(3, TimeUnit.SECONDS)) ;
        } else {
            executeSqlImport(srcDS, targetDSConnection, sql.toString(), dbMapping, successCount, errMsg, debug);
        }
        logger.info(dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
        etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条");
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
    }
    if (errMsg.isEmpty()) {
        etlResult.setSucceeded(true);
    } else {
        etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
    }
    return etlResult;
}

Example 7 with DbMapping

use of com.alibaba.otter.canal.client.adapter.phoenix.config.MappingConfig.DbMapping in project canal by alibaba.

the class PhoenixSyncService method truncate.

/**
 * truncate操作   没有改动
 *
 * @param config MappingConfig
 */
private void truncate(BatchExecutor batchExecutor, MappingConfig config) throws SQLException {
    DbMapping dbMapping = config.getDbMapping();
    StringBuilder sql = new StringBuilder();
    sql.append("TRUNCATE TABLE ").append(SyncUtil.getDbTableName(dbMapping));
    batchExecutor.execute(sql.toString(), new ArrayList<>());
    if (logger.isTraceEnabled()) {
        logger.trace("Truncate target table, sql: {}", sql);
    }
}