Examples with ESSyncConfig com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig used on opensource projects

Example 21 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class RoleSyncJoinOneTest method test03.

/**
 * 主表更新外键值
 */
@Test
public void test03() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "delete from role where id=2");
    Common.sqlExe(ds, "insert into role (id,role_name) values (2,'operator')");
    Common.sqlExe(ds, "update user set role_id=2 where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("role_id", 2L);
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("role_id", 1L);
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("operator", response.getSource().get("_role_name"));
    Common.sqlExe(ds, "update user set role_id=1 where id=1");
    Dml dml2 = new Dml();
    dml2.setDestination("example");
    dml2.setTs(new Date().getTime());
    dml2.setType("UPDATE");
    dml2.setDatabase("mytest");
    dml2.setTable("user");
    List<Map<String, Object>> dataList2 = new ArrayList<>();
    Map<String, Object> data2 = new LinkedHashMap<>();
    dataList2.add(data2);
    data2.put("id", 1L);
    data2.put("role_id", 1L);
    dml2.setData(dataList2);
    List<Map<String, Object>> oldList2 = new ArrayList<>();
    Map<String, Object> old2 = new LinkedHashMap<>();
    oldList2.add(old2);
    old2.put("role_id", 2L);
    dml2.setOld(oldList2);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml2);
    GetResponse response2 = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("admin2", response2.getSource().get("_role_name"));
}

Example 22 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class LabelSyncJoinSub2Test method test03.

/**
 * 带函数子查询从表删除
 */
@Test
public void test03() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "delete from label where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("DELETE");
    dml.setDatabase("mytest");
    dml.setTable("label");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("user_id", 1L);
    data.put("label", "a");
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("b_", response.getSource().get("_labels"));
}

Example 23 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class RoleSyncJoinOne2Test method test02.

/**
 * 带函数非子查询从表更新
 */
@Test
public void test02() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "update role set role_name='admin3' where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("role");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("role_name", "admin3");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("role_name", "admin");
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("admin3_", response.getSource().get("_role_name"));
}

Example 24 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class UserSyncJoinOneTest method test02.

/**
 * 主表带函数更新
 */
@Test
public void test02() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "update user set name='Eric2' where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("name", "Eric2");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("name", "Eric");
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("Eric2_", response.getSource().get("_name"));
}

Example 25 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class ES6xTemplate method updateByQuery.

@Override
public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
    if (paramsTmp.isEmpty()) {
        return;
    }
    ESMapping mapping = config.getEsMapping();
    BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
    paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)));
    // 查询sql批量更新
    DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
    StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE ");
    List<Object> values = new ArrayList<>();
    paramsTmp.forEach((fieldName, value) -> {
        sql.append("_v.").append(fieldName).append("=? AND ");
        values.add(value);
    });
    // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
    int len = sql.length();
    sql.delete(len - 4, len);
    Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> {
        int count = 0;
        try {
            while (rs.next()) {
                Object idVal = getIdValFromRS(mapping, rs);
                append4Update(mapping, idVal, esFieldData);
                commitBulk();
                count++;
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return count;
    });
    if (logger.isTraceEnabled()) {
        logger.trace("Update ES by query affected {} records", syncCount);
    }
}