Examples with Dml com.alibaba.otter.canal.client.adapter.support.Dml used on opensource projects

Example 21 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class RoleSyncJoinOne2Test method test02.

/**
 * 带函数非子查询从表更新
 */
@Test
public void test02() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "update role set role_name='admin3' where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("role");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("role_name", "admin3");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("role_name", "admin");
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("admin3_", response.getSource().get("_role_name"));
}

Example 22 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class UserSyncJoinOneTest method test02.

/**
 * 主表带函数更新
 */
@Test
public void test02() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "update user set name='Eric2' where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("name", "Eric2");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("name", "Eric");
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("Eric2_", response.getSource().get("_name"));
}

Example 23 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class LabelSyncJoinSubTest method test02.

/**
 * 子查询从表更新
 */
@Test
public void test02() {
    DataSource ds = DatasourceConfig.DATA_SOURCES.get("defaultDS");
    Common.sqlExe(ds, "update label set label='aa' where id=1");
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("label");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("user_id", 1L);
    data.put("label", "aa");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("label", "a");
    dml.setOld(oldList);
    String database = dml.getDatabase();
    String table = dml.getTable();
    Map<String, ESSyncConfig> esSyncConfigs = esAdapter.getDbTableEsSyncConfig().get(database + "-" + table);
    esAdapter.getEsSyncService().sync(esSyncConfigs.values(), dml);
    GetResponse response = esAdapter.getEsConnection().getTransportClient().prepareGet("mytest_user", "_doc", "1").get();
    Assert.assertEquals("b;aa", response.getSource().get("_labels"));
}

Example 24 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class TablestoreAdapter method sync.

@Override
public void sync(List<Dml> dmls) {
    if (dmls == null || dmls.isEmpty()) {
        return;
    }
    try {
        Set<TableStoreWriter> writerSet = new HashSet<>();
        List<Future<WriterResult>> futureList = new ArrayList<>();
        for (Dml dml : dmls) {
            String destination = StringUtils.trimToEmpty(dml.getDestination());
            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
            String database = dml.getDatabase();
            String table = dml.getTable();
            String key;
            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
                key = destination + "-" + groupId + "_" + database + "-" + table;
            } else {
                key = destination + "_" + database + "-" + table;
            }
            Map<String, MappingConfig> configMap = mappingConfigCache.get(key);
            if (configMap == null) {
                // 可能有dml中涉及到的表并没有出现在配置中，说明此类dml并不需要同步
                continue;
            }
            Map<String, TableStoreWriter> writerMap = writerCache.get(key);
            for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
                TableStoreWriter w = writerMap.get(entry.getKey());
                // 拿到所有future用于判定失败的记录
                Future<WriterResult> futureTemp = tablestoreSyncService.sync(entry.getValue(), dml, w);
                if (futureTemp != null) {
                    writerSet.add(w);
                    futureList.add(futureTemp);
                }
            }
        }
        if (writerSet.isEmpty()) {
            return;
        }
        writerSet.forEach(e -> e.flush());
        List<WriterResult.RowChangeStatus> totalFailedRows = new ArrayList<>();
        for (Future<WriterResult> future : futureList) {
            try {
                WriterResult result = future.get();
                List<WriterResult.RowChangeStatus> failedRows = result.getFailedRows();
                if (!CollectionUtils.isEmpty(failedRows)) {
                    totalFailedRows.addAll(failedRows);
                }
            } catch (InterruptedException e) {
                logger.info("InterruptedException", e);
                Thread.currentThread().interrupt();
            } catch (ExecutionException e) {
                throw new RuntimeException(e);
            }
        }
        if (!CollectionUtils.isEmpty(totalFailedRows)) {
            // 认为有失败的请求
            List<String> msgs = totalFailedRows.stream().map(e -> buildErrorMsgForFailedRowChange(e)).collect(Collectors.toList());
            throw new RuntimeException("Failed rows:" + org.springframework.util.StringUtils.collectionToDelimitedString(msgs, ",", "[", "]"));
        }
    } catch (Exception e) {
        throw e;
    }
}

Example 25 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class RdbSyncService method sync.

/**
 * 批量同步回调
 *
 * @param dmls 批量 DML
 * @param function 回调方法
 */
public void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
    try {
        boolean toExecute = false;
        for (Dml dml : dmls) {
            if (!toExecute) {
                toExecute = function.apply(dml);
            } else {
                function.apply(dml);
            }
        }
        if (toExecute) {
            List<Future<Boolean>> futures = new ArrayList<>();
            for (int i = 0; i < threads; i++) {
                int j = i;
                if (dmlsPartition[j].isEmpty()) {
                    // bypass
                    continue;
                }
                futures.add(executorThreads[i].submit(() -> {
                    try {
                        dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
                        dmlsPartition[j].clear();
                        batchExecutors[j].commit();
                        return true;
                    } catch (Throwable e) {
                        dmlsPartition[j].clear();
                        batchExecutors[j].rollback();
                        throw new RuntimeException(e);
                    }
                }));
            }
            futures.forEach(future -> {
                try {
                    future.get();
                } catch (ExecutionException | InterruptedException e) {
                    throw new RuntimeException(e);
                }
            });
        }
    } finally {
        for (BatchExecutor batchExecutor : batchExecutors) {
            if (batchExecutor != null) {
                batchExecutor.close();
            }
        }
    }
}