Examples with ESSyncConfig com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig used on opensource projects

Example 6 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class ES7xAdapter method etl.

@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    ESSyncConfig config = esSyncConfig.get(task);
    if (config != null) {
        DataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
        ESEtlService esEtlService = new ESEtlService(esConnection, config);
        if (dataSource != null) {
            return esEtlService.importData(params);
        } else {
            etlResult.setSucceeded(false);
            etlResult.setErrorMessage("DataSource not found");
            return etlResult;
        }
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSuccess = true;
        for (ESSyncConfig configTmp : esSyncConfig.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                ESEtlService esEtlService = new ESEtlService(esConnection, configTmp);
                EtlResult etlRes = esEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSuccess = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSuccess);
            if (resSuccess) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    etlResult.setSucceeded(false);
    etlResult.setErrorMessage("Task not found");
    return etlResult;
}

Example 7 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class ESAdapter method init.

@Override
public void init(OuterAdapterConfig configuration, Properties envProperties) {
    try {
        this.envProperties = envProperties;
        Map<String, ESSyncConfig> esSyncConfigTmp = ESSyncConfigLoader.load(envProperties);
        // 过滤不匹配的key的配置
        esSyncConfigTmp.forEach((key, config) -> {
            if ((config.getOuterAdapterKey() == null && configuration.getKey() == null) || (config.getOuterAdapterKey() != null && config.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
                esSyncConfig.put(key, config);
            }
        });
        for (Map.Entry<String, ESSyncConfig> entry : esSyncConfig.entrySet()) {
            String configName = entry.getKey();
            ESSyncConfig config = entry.getValue();
            addSyncConfigToCache(configName, config);
        }
        esSyncService = new ESSyncService(esTemplate);
        esConfigMonitor = new ESConfigMonitor();
        esConfigMonitor.init(this, envProperties);
    } catch (Throwable e) {
        throw new RuntimeException(e);
    }
}

Example 8 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class ESAdapter method addSyncConfigToCache.

public void addSyncConfigToCache(String configName, ESSyncConfig config) {
    Properties envProperties = this.envProperties;
    SchemaItem schemaItem = SqlParser.parse(config.getEsMapping().getSql());
    config.getEsMapping().setSchemaItem(schemaItem);
    DruidDataSource dataSource = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
    if (dataSource == null || dataSource.getUrl() == null) {
        throw new RuntimeException("No data source found: " + config.getDataSourceKey());
    }
    Pattern pattern = Pattern.compile(".*:(.*)://.*/(.*)\\?.*$");
    Matcher matcher = pattern.matcher(dataSource.getUrl());
    if (!matcher.find()) {
        throw new RuntimeException("Not found the schema of jdbc-url: " + config.getDataSourceKey());
    }
    String schema = matcher.group(2);
    schemaItem.getAliasTableItems().values().forEach(tableItem -> {
        Map<String, ESSyncConfig> esSyncConfigMap;
        if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
            esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "-" + StringUtils.trimToEmpty(config.getGroupId()) + "_" + schema + "-" + tableItem.getTableName(), k -> new ConcurrentHashMap<>());
        } else {
            esSyncConfigMap = dbTableEsSyncConfig.computeIfAbsent(StringUtils.trimToEmpty(config.getDestination()) + "_" + schema + "-" + tableItem.getTableName(), k -> new ConcurrentHashMap<>());
        }
        esSyncConfigMap.put(configName, config);
    });
}

Example 9 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class LabelSyncJoinSubTest method test01.

/**
 * 子查询从表插入
 */
@Test
public void test01() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "delete from label where id=1 or id=2");
    Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
    Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("INSERT");
    dml.setDatabase("mytest");
    dml.setTable("label");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 2L);
    data.put("user_id", 1L);
    data.put("label", "b");
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("b;a", response.getSource().get("_labels"));
}

Example 10 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class RoleSyncJoinOneTest method test01.

/**
 * 非子查询从表插入
 */
@Test
public void test01() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "delete from role where id=1");
    Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("INSERT");
    dml.setDatabase("mytest");
    dml.setTable("role");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("role_name", "admin");
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("admin", response.getSource().get("_role_name"));
}