Examples with EtlResult com.alibaba.otter.canal.client.adapter.support.EtlResult used on opensource projects

Example 6 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class HbaseAdapter method etl.

@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    MappingConfig config = hbaseMapping.get(task);
    HbaseEtlService hbaseEtlService = new HbaseEtlService(hbaseTemplate, config);
    if (config != null) {
        return hbaseEtlService.importData(params);
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSucc = true;
        for (MappingConfig configTmp : hbaseMapping.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                EtlResult etlRes = hbaseEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSucc = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSucc);
            if (resSucc) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    etlResult.setSucceeded(false);
    etlResult.setErrorMessage("Task not found");
    return etlResult;
}

Example 7 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class TablestoreAdapter method etl.

@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    MappingConfig config = tablestoreMapping.get(task);
    if (config == null) {
        etlResult.setErrorMessage("can not find config for " + task);
        etlResult.setSucceeded(false);
        return etlResult;
    }
    TableStoreWriter writer = null;
    try {
        writer = buildEtlWriter(configuration, config);
        TablestoreEtlService rdbEtlService = new TablestoreEtlService(writer, config);
        rdbEtlService.importData(params);
        etlResult.setSucceeded(true);
        return etlResult;
    } catch (Exception e) {
        logger.error("Error while etl for task " + task, e);
        etlResult.setSucceeded(false);
        etlResult.setErrorMessage(e.getMessage());
        return etlResult;
    } finally {
        if (writer != null) {
            writer.close();
        }
    }
}

Example 8 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class KuduAdapter method etl.

@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    KuduMappingConfig config = kuduMapping.get(task);
    KuduEtlService hbaseEtlService = new KuduEtlService(kuduTemplate, config);
    if (config != null) {
        return hbaseEtlService.importData(params);
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSucc = true;
        for (KuduMappingConfig configTmp : kuduMapping.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                EtlResult etlRes = hbaseEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSucc = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSucc);
            if (resSucc) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    return null;
}

Example 9 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class PhoenixEtlService method importData.

/**
 * 导入数据
 */
public static EtlResult importData(DataSource srcDS, Connection targetDSConnection, MappingConfig config, List<String> params) {
    EtlResult etlResult = new EtlResult();
    AtomicLong successCount = new AtomicLong();
    List<String> errMsg = new ArrayList<>();
    String hbaseTable = "";
    try {
        if (config == null) {
            logger.error("Config is null!");
            etlResult.setSucceeded(false);
            etlResult.setErrorMessage("Config is null!");
            return etlResult;
        }
        boolean debug = params != null && params.get(0).equals("_debug");
        if (debug) {
            params = params.subList(1, params.size());
        }
        syncSchema(srcDS, targetDSConnection, config);
        DbMapping dbMapping = config.getDbMapping();
        long start = System.currentTimeMillis();
        // 拼接sql
        StringBuilder sql = new StringBuilder("SELECT * FROM " + dbMapping.getDatabase() + "." + dbMapping.getTable());
        // 拼接条件
        appendCondition(params, dbMapping, srcDS, sql);
        // 获取总数
        String countSql = "SELECT COUNT(1) FROM ( " + sql + ") _CNT ";
        long cnt = (Long) Util.sqlRS(srcDS, countSql, rs -> {
            Long count = null;
            try {
                if (rs.next()) {
                    count = ((Number) rs.getObject(1)).longValue();
                }
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
            }
            return count == null ? 0 : count;
        });
        // 当大于1万条记录时开启多线程
        if (cnt >= 10000) {
            int threadCount = 3;
            long perThreadCnt = cnt / threadCount;
            ExecutorService executor = Util.newFixedThreadPool(threadCount, 5000L);
            for (int i = 0; i < threadCount; i++) {
                long offset = i * perThreadCnt;
                Long size = null;
                if (i != threadCount - 1) {
                    size = perThreadCnt;
                }
                String sqlFinal;
                if (size != null) {
                    sqlFinal = sql + " LIMIT " + offset + "," + size;
                } else {
                    sqlFinal = sql + " LIMIT " + offset + "," + cnt;
                }
                executor.execute(() -> executeSqlImport(srcDS, targetDSConnection, sqlFinal, dbMapping, successCount, errMsg, debug));
            }
            executor.shutdown();
            // noinspection StatementWithEmptyBody
            while (!executor.awaitTermination(3, TimeUnit.SECONDS)) ;
        } else {
            executeSqlImport(srcDS, targetDSConnection, sql.toString(), dbMapping, successCount, errMsg, debug);
        }
        logger.info(dbMapping.getTable() + " etl completed in: " + (System.currentTimeMillis() - start) / 1000 + "s!");
        etlResult.setResultMessage("导入目标表 " + SyncUtil.getDbTableName(dbMapping) + " 数据：" + successCount.get() + " 条");
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        errMsg.add(hbaseTable + " etl failed! ==>" + e.getMessage());
    }
    if (errMsg.isEmpty()) {
        etlResult.setSucceeded(true);
    } else {
        etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
    }
    return etlResult;
}

Example 10 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class KuduEtlService method importData.

public EtlResult importData(List<String> params) {
    EtlResult etlResult = new EtlResult();
    List<String> errMsg = new ArrayList<>();
    KuduMappingConfig.KuduMapping kuduMapping = config.getKuduMapping();
    boolean flag = kuduTemplate.tableExists(kuduMapping.getTargetTable());
    // 表不存在，停止导入
    if (!flag) {
        logger.info("{} is don't hava,please check your kudu table !", kuduMapping.getTargetTable());
        errMsg.add(kuduMapping.getTargetTable() + " is don't hava,please check your kudu table !");
        etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
        return etlResult;
    }
    logger.info("{} etl is starting!", kuduMapping.getTargetTable());
    String sql = "SELECT * FROM " + kuduMapping.getDatabase() + "." + kuduMapping.getTable();
    return importData(sql, params);
}