Examples with Dml com.alibaba.otter.canal.client.adapter.support.Dml used on opensource projects

Example 1 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class KuduAdapter method sync.

@Override
public void sync(List<Dml> dmls) {
    if (dmls == null || dmls.isEmpty()) {
        return;
    }
    for (Dml dml : dmls) {
        if (dml == null) {
            return;
        }
        String destination = StringUtils.trimToEmpty(dml.getDestination());
        String groupId = StringUtils.trimToEmpty(dml.getGroupId());
        String database = dml.getDatabase();
        String table = dml.getTable();
        Map<String, KuduMappingConfig> configMap;
        if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
            configMap = mappingConfigCache.get(destination + "-" + groupId + "_" + database + "-" + table);
        } else {
            configMap = mappingConfigCache.get(destination + "_" + database + "-" + table);
        }
        if (configMap != null) {
            List<KuduMappingConfig> configs = new ArrayList<>();
            configMap.values().forEach(config -> {
                if (StringUtils.isNotEmpty(config.getGroupId())) {
                    if (config.getGroupId().equals(dml.getGroupId())) {
                        configs.add(config);
                    }
                } else {
                    configs.add(config);
                }
            });
            if (!configs.isEmpty()) {
                configs.forEach(config -> kuduSyncService.sync(config, dml));
            } else {
                logger.error("groupID didn't mach,please check your gruopId ");
            }
        } else {
            logger.error("{} config didn't get,please check your map key ", destination + "_" + database + "-" + table);
        }
    }
}

Example 2 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class AdapterProcessor method batchSync.

/**
 * 分批同步
 *
 * @param dmls
 * @param adapter
 */
private void batchSync(List<Dml> dmls, OuterAdapter adapter) {
    // 分批同步
    if (dmls.size() <= canalClientConfig.getSyncBatchSize()) {
        adapter.sync(dmls);
    } else {
        int len = 0;
        List<Dml> dmlsBatch = new ArrayList<>();
        for (Dml dml : dmls) {
            dmlsBatch.add(dml);
            if (dml.getData() == null || dml.getData().isEmpty()) {
                len += 1;
            } else {
                len += dml.getData().size();
            }
            if (len >= canalClientConfig.getSyncBatchSize()) {
                adapter.sync(dmlsBatch);
                dmlsBatch.clear();
                len = 0;
            }
        }
        if (!dmlsBatch.isEmpty()) {
            adapter.sync(dmlsBatch);
        }
    }
}

Example 3 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class PhoenixSyncService method sync.

/**
 * 批量同步回调
 *
 * @param dmls     批量 DML
 * @param function 回调方法
 */
private void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
    try {
        boolean toExecute = false;
        for (Dml dml : dmls) {
            if (!toExecute) {
                toExecute = function.apply(dml);
            } else {
                function.apply(dml);
            }
        }
        if (toExecute) {
            List<Future<Boolean>> futures = new ArrayList<>();
            for (int i = 0; i < threads; i++) {
                int j = i;
                if (dmlsPartition[j].isEmpty()) {
                    // bypass
                    continue;
                }
                futures.add(executorThreads[i].submit(() -> {
                    try {
                        dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
                        // 相对于RDB同步 少了  dmlsPartition[j].clear();
                        // 在 try catch中获取异常后再次执行一次batchExecutors[j].commit()
                        batchExecutors[j].commit();
                        return true;
                    } catch (Throwable e) {
                        batchExecutors[j].rollback();
                        if (!e.getClass().getName().endsWith("ColumnNotFoundException") && !e.getClass().getName().endsWith("TableNotFoundException")) {
                            throw new RuntimeException(e);
                        }
                        logger.info("table or column not found: " + e.getMessage());
                        boolean synced = false;
                        for (SyncItem syncItem : dmlsPartition[j]) {
                            if (PhoenixEtlService.syncSchema(batchExecutors[j].getConn(), syncItem.config)) {
                                synced = true;
                            }
                        }
                        if (!synced) {
                            throw new RuntimeException(e);
                        }
                        dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
                        try {
                            batchExecutors[j].commit();
                            return true;
                        } catch (Throwable e1) {
                            batchExecutors[j].rollback();
                            throw new RuntimeException(e1);
                        }
                    } finally {
                        dmlsPartition[j].clear();
                    }
                }));
            }
            futures.forEach(future -> {
                try {
                    future.get();
                } catch (ExecutionException | InterruptedException e) {
                    throw new RuntimeException(e);
                }
            });
        }
    } finally {
        for (BatchExecutor batchExecutor : batchExecutors) {
            if (batchExecutor != null) {
                batchExecutor.close();
            }
        }
    }
}

Example 4 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class PhoenixSyncTest method test02.

@Test
public void test02() {
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("UPDATE");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1);
    data.put("name", "sixPulseExcalibur2");
    dml.setData(dataList);
    List<Map<String, Object>> oldList = new ArrayList<>();
    Map<String, Object> old = new LinkedHashMap<>();
    oldList.add(old);
    old.put("name", "sixPulseExcalibur");
    dml.setOld(oldList);
    phoenixAdapter.sync(Collections.singletonList(dml));
}

Example 5 with Dml

use of com.alibaba.otter.canal.client.adapter.support.Dml in project canal by alibaba.

the class OracleSyncTest method test01.

@Test
public void test01() {
    Dml dml = new Dml();
    dml.setDestination("example");
    dml.setTs(new Date().getTime());
    dml.setType("INSERT");
    dml.setDatabase("mytest");
    dml.setTable("user");
    List<Map<String, Object>> dataList = new ArrayList<>();
    Map<String, Object> data = new LinkedHashMap<>();
    dataList.add(data);
    data.put("id", 1L);
    data.put("name", "Eric");
    data.put("role_id", 1L);
    data.put("c_time", new Date());
    data.put("test1", "sdfasdfawe中国asfwef");
    dml.setData(dataList);
    rdbAdapter.sync(Collections.singletonList(dml));
}