Examples with SingleDml com.alibaba.otter.canal.client.adapter.phoenix.support.SingleDml used on opensource projects

Example 1 with SingleDml

use of com.alibaba.otter.canal.client.adapter.phoenix.support.SingleDml in project canal by alibaba.

the class PhoenixSyncService method sync.

/**
 * 批量同步 :将批量DML进行解析并放入List<SingleDml> --> dmlsPartition[hash].add(syncItem);
 * @param mappingConfig 配置集合
 * @param dmls   批量 DML
 */
public void sync(Map<String, Map<String, MappingConfig>> mappingConfig, List<Dml> dmls, Properties envProperties) {
    sync(dmls, dml -> {
        String destination = StringUtils.trimToEmpty(dml.getDestination());
        String groupId = StringUtils.trimToEmpty(dml.getGroupId());
        String database = dml.getDatabase();
        String table = dml.getTable().toLowerCase();
        Map<String, MappingConfig> configMap;
        if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
            // tcp 模式
            configMap = mappingConfig.get(destination + "-" + groupId + "_" + database + "-" + table);
        } else {
            // kafka 模式 或者 RocketMQ模式
            configMap = mappingConfig.get(destination + "_" + database + "-" + table);
        }
        if (configMap == null) {
            if (logger.isTraceEnabled()) {
                logger.trace("no config map: destination={},groupId={}, database={}, table={}, keys={}", destination, groupId, database, table, mappingConfig.keySet());
            }
            return false;
        }
        if (configMap.values().isEmpty()) {
            logger.info("config map empty: destination={},groupId={}, database={}, table={}, keys={}", destination, groupId, database, table, mappingConfig.keySet());
            return false;
        }
        if (dml.getIsDdl() != null && dml.getIsDdl() && StringUtils.isNotEmpty(dml.getSql())) {
            // DDL
            columnsTypeCache.remove(dml.getDestination() + "." + dml.getDatabase() + "." + dml.getTable());
            List<SQLStatement> stmtList;
            try {
                stmtList = SQLUtils.parseStatements(dml.getSql(), JdbcConstants.MYSQL, false);
            } catch (ParserException e) {
                // 可能存在一些SQL是不支持的，比如存储过程
                logger.info("parse sql error: " + dml.getSql(), e);
                return false;
            }
            for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
                try {
                    alter(batchExecutors[0], entry.getValue(), dml, stmtList, entry.getKey());
                } catch (SQLException e) {
                    throw new RuntimeException(e);
                }
            }
            return false;
        } else {
            // DML
            for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
                MappingConfig config = entry.getValue();
                if (config.isDebug()) {
                    logger.info("DML: {} {}", entry.getKey(), JSON.toJSONString(dml, SerializerFeature.WriteMapNullValue));
                }
                if (config.getConcurrent()) {
                    // 并行同步
                    // 将一批DML转成SingleDml
                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
                    singleDmls.forEach(singleDml -> {
                        // 取主键hash
                        int hash = pkHash(config.getDbMapping(), singleDml.getData());
                        SyncItem syncItem = new SyncItem(config, singleDml);
                        // 相同的主键数据的顺序是可以保证的
                        dmlsPartition[hash].add(syncItem);
                    });
                } else {
                    // 不并行同步
                    int hash = 0;
                    List<SingleDml> singleDmls = SingleDml.dml2SingleDmls(dml);
                    singleDmls.forEach(singleDml -> {
                        SyncItem syncItem = new SyncItem(config, singleDml);
                        // 这里线程默认是3个,如果不并行，则会出现2个线程空跑
                        dmlsPartition[hash].add(syncItem);
                    });
                }
            }
            return true;
        }
    });
}