Examples with EtlResult com.alibaba.otter.canal.client.adapter.support.EtlResult used on opensource projects

Example 1 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class HbaseEtlService method importData.

/**
 * 导入数据
 *
 * @param params 筛选条件
 * @return 导入结果
 */
public EtlResult importData(List<String> params) {
    EtlResult etlResult = new EtlResult();
    List<String> errMsg = new ArrayList<>();
    try {
        MappingConfig.HbaseMapping hbaseMapping = config.getHbaseMapping();
        if (params != null && params.size() == 1 && "rebuild".equalsIgnoreCase(params.get(0))) {
            logger.info(hbaseMapping.getHbaseTable() + " rebuild is starting!");
            // 如果表存在则删除
            if (hbaseTemplate.tableExists(hbaseMapping.getHbaseTable())) {
                hbaseTemplate.disableTable(hbaseMapping.getHbaseTable());
                hbaseTemplate.deleteTable(hbaseMapping.getHbaseTable());
            }
            params = null;
        } else {
            logger.info(hbaseMapping.getHbaseTable() + " etl is starting!");
        }
        createTable();
        // 拼接sql
        String sql = "SELECT * FROM `" + config.getHbaseMapping().getDatabase() + "`.`" + hbaseMapping.getTable() + "`";
        return super.importData(sql, params);
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        errMsg.add("HBase etl error ==>" + e.getMessage());
    }
    etlResult.setErrorMessage(Joiner.on("\n").join(errMsg));
    return etlResult;
}

Example 2 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class CommonRest method etl.

/**
 * ETL curl http://127.0.0.1:8081/etl/rdb/oracle1/mytest_user.yml -X POST
 *
 * @param type 类型 hbase, es
 * @param key adapter key
 * @param task 任务名对应配置文件名 mytest_user.yml
 * @param params etl where条件参数, 为空全部导入
 */
@PostMapping("/etl/{type}/{key}/{task}")
public EtlResult etl(@PathVariable String type, @PathVariable String key, @PathVariable String task, @RequestParam(name = "params", required = false) String params) {
    OuterAdapter adapter = loader.getExtension(type, key);
    String destination = adapter.getDestination(task);
    String lockKey = destination == null ? task : destination;
    boolean locked = etlLock.tryLock(ETL_LOCK_ZK_NODE + type + "-" + lockKey);
    if (!locked) {
        EtlResult result = new EtlResult();
        result.setSucceeded(false);
        result.setErrorMessage(task + " 有其他进程正在导入中, 请稍后再试");
        return result;
    }
    try {
        boolean oriSwitchStatus;
        if (destination != null) {
            oriSwitchStatus = syncSwitch.status(destination);
            if (oriSwitchStatus) {
                syncSwitch.off(destination);
            }
        } else {
            // task可能为destination，直接锁task
            oriSwitchStatus = syncSwitch.status(task);
            if (oriSwitchStatus) {
                syncSwitch.off(task);
            }
        }
        try {
            List<String> paramArray = null;
            if (params != null) {
                paramArray = Arrays.asList(params.trim().split(";"));
            }
            return adapter.etl(task, paramArray);
        } finally {
            if (destination != null && oriSwitchStatus) {
                syncSwitch.on(destination);
            } else if (destination == null && oriSwitchStatus) {
                syncSwitch.on(task);
            }
        }
    } finally {
        etlLock.unlock(ETL_LOCK_ZK_NODE + type + "-" + lockKey);
    }
}

Example 3 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class RdbAdapter method etl.

/**
 * ETL方法
 *
 * @param task 任务名, 对应配置名
 * @param params etl筛选条件
 * @return ETL结果
 */
@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    MappingConfig config = rdbMapping.get(task);
    RdbEtlService rdbEtlService = new RdbEtlService(dataSource, config);
    if (config != null) {
        return rdbEtlService.importData(params);
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSucc = true;
        for (MappingConfig configTmp : rdbMapping.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                EtlResult etlRes = rdbEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSucc = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSucc);
            if (resSucc) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    etlResult.setSucceeded(false);
    etlResult.setErrorMessage("Task not found");
    return etlResult;
}

Example 4 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class ES7xAdapter method etl.

@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    ESSyncConfig config = esSyncConfig.get(task);
    if (config != null) {
        DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
        ESEtlService esEtlService = new ESEtlService(esConnection, config);
        if (dataSource != null) {
            return esEtlService.importData(params);
        } else {
            etlResult.setSucceeded(false);
            etlResult.setErrorMessage("DataSource not found");
            return etlResult;
        }
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSuccess = true;
        for (ESSyncConfig configTmp : esSyncConfig.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
                EtlResult etlRes = esEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSuccess = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSuccess);
            if (resSuccess) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    etlResult.setSucceeded(false);
    etlResult.setErrorMessage("Task not found");
    return etlResult;
}

Example 5 with EtlResult

use of com.alibaba.otter.canal.client.adapter.support.EtlResult in project canal by alibaba.

the class ES6xAdapter method etl.

@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    ESSyncConfig config = esSyncConfig.get(task);
    if (config != null) {
        DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
        ESEtlService esEtlService = new ESEtlService(esConnection, config);
        if (dataSource != null) {
            return esEtlService.importData(params);
        } else {
            etlResult.setSucceeded(false);
            etlResult.setErrorMessage("DataSource not found");
            return etlResult;
        }
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSuccess = true;
        for (ESSyncConfig configTmp : esSyncConfig.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
                EtlResult etlRes = esEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSuccess = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSuccess);
            if (resSuccess) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    etlResult.setSucceeded(false);
    etlResult.setErrorMessage("Task not found");
    return etlResult;
}