Examples with ESSyncConfig com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig used on opensource projects

Example 1 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class ESSyncService method sync.

public void sync(Collection<ESSyncConfig> esSyncConfigs, Dml dml) {
    long begin = System.currentTimeMillis();
    if (esSyncConfigs != null) {
        if (logger.isTraceEnabled()) {
            logger.trace("Destination: {}, database:{}, table:{}, type:{}, affected index count: {}", dml.getDestination(), dml.getDatabase(), dml.getTable(), dml.getType(), esSyncConfigs.size());
        }
        for (ESSyncConfig config : esSyncConfigs) {
            if (logger.isTraceEnabled()) {
                logger.trace("Prepared to sync index: {}, destination: {}", config.getEsMapping().get_index(), dml.getDestination());
            }
            this.sync(config, dml);
            if (logger.isTraceEnabled()) {
                logger.trace("Sync completed: {}, destination: {}", config.getEsMapping().get_index(), dml.getDestination());
            }
        }
        if (logger.isTraceEnabled()) {
            logger.trace("Sync elapsed time: {} ms, affected indexes count：{}, destination: {}", (System.currentTimeMillis() - begin), esSyncConfigs.size(), dml.getDestination());
        }
        if (logger.isDebugEnabled()) {
            StringBuilder configIndexes = new StringBuilder();
            esSyncConfigs.forEach(esSyncConfig -> configIndexes.append(esSyncConfig.getEsMapping().get_index()).append(" "));
            logger.debug("DML: {} \nAffected indexes: {}", JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue), configIndexes.toString());
        }
    }
}

Example 2 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class LabelSyncJoinSub2Test method test01.

/**
 * 带函数子查询从表插入
 */
@Test
public void test01() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "delete from label where id=1 or id=2");
    Common.sqlExe(ds, "insert into label (id,user_id,label) values (1,1,'a')");
    Common.sqlExe(ds, "insert into label (id,user_id,label) values (2,1,'b')");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("INSERT");
    dml.setDatabase("mytest");
    dml.setTable("label");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 2L);
    data.put("user_id", 1L);
    data.put("label", "b");
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("b;a_", response.getSource().get("_labels"));
}

Example 3 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class LabelSyncJoinSub2Test method test02.

/**
 * 带函数子查询从表更新
 */
@Test
public void test02() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "update label set label='aa' where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("label");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("user_id", 1L);
    data.put("label", "aa");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("label", "v");
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("b;aa_", response.getSource().get("_labels"));
}

Example 4 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class RoleSyncJoinOne2Test method test01.

/**
 * 带函数非子查询从表插入
 */
@Test
public void test01() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "delete from role where id=1");
    Common.sqlExe(ds, "insert into role (id,role_name) values (1,'admin')");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("INSERT");
    dml.setDatabase("mytest");
    dml.setTable("role");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("role_name", "admin");
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("admin_", response.getSource().get("_role_name"));
}

Example 5 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class UserSyncJoinOneTest method test01.

/**
 * 主表带函数插入
 */
@Test
public void test01() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "delete from user where id=1");
    Common.sqlExe(ds, "insert into user (id,name,role_id) values (1,'Eric',1)");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("INSERT");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("name", "Eric");
    data.put("role_id", 1L);
    data.put("c_time", new Date());
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("Eric_", response.getSource().get("_name"));
}