Examples with ESSyncConfig com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig used on opensource projects

Example 16 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class ES7xAdapter method count.

@Override
public Map<String, Object> count(String task) {
    ESSyncConfig config = esSyncConfig.get(task);
    ESSyncConfig.ESMapping mapping = config.getEsMapping();
    SearchResponse response = this.esConnection.new ESSearchRequest(mapping.get_index()).size(0).getResponse();
    long rowCount = response.getHits().getTotalHits().value;
    Map<String, Object> res = new LinkedHashMap<>();
    res.put("esIndex", mapping.get_index());
    res.put("count", rowCount);
    return res;
}

Example 17 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class UserSyncSingleTest method test02.

/**
 * 单表更新
 */
@Test
public void test02() {
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("name", "Eric2");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("name", "Eric");
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("Eric2", response.getSource().get("_name"));
}

Example 18 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class UserSyncSingleTest method test01.

/**
 * 单表插入
 */
@Test
public void test01() {
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("INSERT");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("name", "Eric");
    data.put("role_id", 1L);
    data.put("c_time", new Date());
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("Eric", response.getSource().get("_name"));
}

Example 19 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class UserSyncSingleTest method test03.

/**
 * 单表删除
 */
@Test
public void test03() {
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("DELETE");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("name", "Eric");
    data.put("role_id", 1L);
    data.put("c_time", new Date());
    dml.setData(dataList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertNull(response.getSource());
}

Example 20 with ESSyncConfig

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig in project canal by alibaba.

the class ES7xTemplate method updateByQuery.

@Override
public void updateByQuery(ESSyncConfig config, Map<String, Object> paramsTmp, Map<String, Object> esFieldData) {
    if (paramsTmp.isEmpty()) {
        return;
    }
    ESMapping mapping = config.getEsMapping();
    BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
    paramsTmp.forEach((fieldName, value) -> queryBuilder.must(QueryBuilders.termsQuery(fieldName, value)));
    // 查询sql批量更新
    DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
    StringBuilder sql = new StringBuilder("SELECT * FROM (" + mapping.getSql() + ") _v WHERE ");
    List<Object> values = new ArrayList<>();
    paramsTmp.forEach((fieldName, value) -> {
        sql.append("_v.").append(fieldName).append("=? AND ");
        values.add(value);
    });
    // TODO 直接外部包裹sql会导致全表扫描性能低, 待优化拼接内部where条件
    int len = sql.length();
    sql.delete(len - 4, len);
    Integer syncCount = (Integer) Util.sqlRS(ds, sql.toString(), values, rs -> {
        int count = 0;
        try {
            while (rs.next()) {
                Object idVal = getIdValFromRS(mapping, rs);
                append4Update(mapping, idVal, esFieldData);
                commitBulk();
                count++;
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return count;
    });
    if (logger.isTraceEnabled()) {
        logger.trace("Update ES by query affected {} records", syncCount);
    }
}