Examples with ESMapping com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping used on opensource projects

Example 11 with ESMapping

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping in project canal by alibaba.

the class ESSyncService method mainTableUpdate.

/**
 * 主表(单表)复杂字段update
 *
 * @param config es配置
 * @param dml dml信息
 * @param data 单行dml数据
 */
private void mainTableUpdate(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old) {
    ESMapping mapping = config.getEsMapping();
    String sql = mapping.getSql();
    String condition = ESSyncUtil.pkConditionSql(mapping, data);
    sql = ESSyncUtil.appendCondition(sql, condition);
    DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
    if (logger.isTraceEnabled()) {
        logger.trace("Main table update to es index by query sql, destination:{}, table: {}, index: {}, sql: {}", config.getDestination(), dml.getTable(), mapping.get_index(), sql.replace("\n", " "));
    }
    Util.sqlRS(ds, sql, rs -> {
        try {
            while (rs.next()) {
                Map<String, Object> esFieldData = new LinkedHashMap<>();
                Object idVal = esTemplate.getESDataFromRS(mapping, rs, old, esFieldData);
                if (logger.isTraceEnabled()) {
                    logger.trace("Main table update to es index by query sql, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), idVal);
                }
                esTemplate.update(mapping, idVal, esFieldData);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return 0;
    });
}

Example 12 with ESMapping

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping in project canal by alibaba.

the class ESSyncService method subTableSimpleFieldOperation.

/**
 * 关联子查询, 主表简单字段operation
 *
 * @param config es配置
 * @param dml dml信息
 * @param data 单行dml数据
 * @param old 单行old数据
 * @param tableItem 当前表配置
 */
private void subTableSimpleFieldOperation(ESSyncConfig config, Dml dml, Map<String, Object> data, Map<String, Object> old, TableItem tableItem) {
    ESMapping mapping = config.getEsMapping();
    MySqlSelectQueryBlock queryBlock = SqlParser.parseSQLSelectQueryBlock(tableItem.getSubQuerySql());
    StringBuilder sql = new StringBuilder();
    sql.append("SELECT ").append(SqlParser.parse4SQLSelectItem(queryBlock)).append(" FROM ").append(SqlParser.parse4FromTableSource(queryBlock));
    String whereSql = SqlParser.parse4WhereItem(queryBlock);
    if (whereSql != null) {
        sql.append(" WHERE ").append(whereSql);
    } else {
        sql.append(" WHERE 1=1 ");
    }
    List<Object> values = new ArrayList<>();
    for (FieldItem fkFieldItem : tableItem.getRelationTableFields().keySet()) {
        String columnName = fkFieldItem.getColumn().getColumnName();
        Object value = esTemplate.getValFromData(mapping, data, fkFieldItem.getFieldName(), columnName);
        sql.append(" AND ").append(columnName).append("=? ");
        values.add(value);
    }
    String groupSql = SqlParser.parse4GroupBy(queryBlock);
    if (groupSql != null) {
        sql.append(groupSql);
    }
    DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
    if (logger.isTraceEnabled()) {
        logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}, sql: {}", config.getDestination(), dml.getTable(), mapping.get_index(), sql.toString().replace("\n", " "));
    }
    Util.sqlRS(ds, sql.toString(), values, rs -> {
        try {
            while (rs.next()) {
                Map<String, Object> esFieldData = new LinkedHashMap<>();
                for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
                    if (old != null) {
                        out: for (FieldItem fieldItem1 : tableItem.getSubQueryFields()) {
                            for (ColumnItem columnItem0 : fieldItem.getColumnItems()) {
                                if (fieldItem1.getFieldName().equals(columnItem0.getColumnName()))
                                    for (ColumnItem columnItem : fieldItem1.getColumnItems()) {
                                        if (old.containsKey(columnItem.getColumnName())) {
                                            Object val = esTemplate.getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getColumn().getColumnName());
                                            esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
                                            break out;
                                        }
                                    }
                            }
                        }
                    } else {
                        Object val = esTemplate.getValFromRS(mapping, rs, fieldItem.getFieldName(), fieldItem.getColumn().getColumnName());
                        esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), val);
                    }
                }
                Map<String, Object> paramsTmp = new LinkedHashMap<>();
                for (Map.Entry<FieldItem, List<FieldItem>> entry : tableItem.getRelationTableFields().entrySet()) {
                    for (FieldItem fieldItem : entry.getValue()) {
                        if (fieldItem.getColumnItems().size() == 1) {
                            Object value = esTemplate.getValFromRS(mapping, rs, fieldItem.getFieldName(), entry.getKey().getColumn().getColumnName());
                            String fieldName = fieldItem.getFieldName();
                            // 判断是否是主键
                            if (fieldName.equals(mapping.get_id())) {
                                fieldName = "_id";
                            }
                            paramsTmp.put(fieldName, value);
                        }
                    }
                }
                if (logger.isDebugEnabled()) {
                    logger.trace("Join table update es index by query sql, destination:{}, table: {}, index: {}", config.getDestination(), dml.getTable(), mapping.get_index());
                }
                esTemplate.updateByQuery(config, paramsTmp, esFieldData);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return 0;
    });
}

Example 13 with ESMapping

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping in project canal by alibaba.

the class ESSyncService method delete.

/**
 * 删除操作dml
 *
 * @param config es配置
 * @param dml dml数据
 */
private void delete(ESSyncConfig config, Dml dml) {
    List<Map<String, Object>> dataList = dml.getData();
    if (dataList == null || dataList.isEmpty()) {
        return;
    }
    SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
    for (Map<String, Object> data : dataList) {
        if (data == null || data.isEmpty()) {
            continue;
        }
        ESMapping mapping = config.getEsMapping();
        // ------是主表------
        if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
            if (mapping.get_id() != null) {
                FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping);
                // 主键为简单字段
                if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) {
                    Object idVal = esTemplate.getValFromData(mapping, data, idFieldItem.getFieldName(), idFieldItem.getColumn().getColumnName());
                    if (logger.isTraceEnabled()) {
                        logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), idVal);
                    }
                    esTemplate.delete(mapping, idVal, null);
                } else {
                    // ------主键带函数, 查询sql获取主键删除------
                    // FIXME 删除时反查sql为空记录, 无法获获取 id field 值
                    mainTableDelete(config, dml, data);
                }
            } else {
                FieldItem pkFieldItem = schemaItem.getIdFieldItem(mapping);
                if (!pkFieldItem.isMethod() && !pkFieldItem.isBinaryOp()) {
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    Object pkVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
                    if (logger.isTraceEnabled()) {
                        logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, pk: {}", config.getDestination(), dml.getTable(), mapping.get_index(), pkVal);
                    }
                    esFieldData.remove(pkFieldItem.getFieldName());
                    esFieldData.keySet().forEach(key -> esFieldData.put(key, null));
                    esTemplate.delete(mapping, pkVal, esFieldData);
                } else {
                    // ------主键带函数, 查询sql获取主键删除------
                    mainTableDelete(config, dml, data);
                }
            }
        }
        // 从表的操作
        for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
            if (tableItem.isMain()) {
                continue;
            }
            if (!tableItem.getTableName().equals(dml.getTable())) {
                continue;
            }
            // 关联条件出现在主表查询条件是否为简单字段
            boolean allFieldsSimple = true;
            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
                if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
                    allFieldsSimple = false;
                    break;
                }
            }
            // 所有查询字段均为简单字段
            if (allFieldsSimple) {
                // 不是子查询
                if (!tableItem.isSubQuery()) {
                    // ------关联表简单字段更新为null------
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
                        esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), null);
                    }
                    joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
                } else {
                    // ------关联子表简单字段更新------
                    subTableSimpleFieldOperation(config, dml, data, null, tableItem);
                }
            } else {
                // ------关联子表复杂字段更新 执行全sql更新es------
                wholeSqlOperation(config, dml, data, null, tableItem);
            }
        }
    }
}

Example 14 with ESMapping

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping in project canal by alibaba.

the class ESSyncService method singleTableSimpleFiledInsert.

/**
 * 单表简单字段insert
 *
 * @param config es配置
 * @param dml dml信息
 * @param data 单行dml数据
 */
private void singleTableSimpleFiledInsert(ESSyncConfig config, Dml dml, Map<String, Object> data) {
    ESMapping mapping = config.getEsMapping();
    Map<String, Object> esFieldData = new LinkedHashMap<>();
    Object idVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
    if (logger.isTraceEnabled()) {
        logger.trace("Single table insert to es index, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), idVal);
    }
    esTemplate.insert(mapping, idVal, esFieldData);
}

Example 15 with ESMapping

use of com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping in project canal by alibaba.

the class ESSyncService method mainTableDelete.

private void mainTableDelete(ESSyncConfig config, Dml dml, Map<String, Object> data) {
    ESMapping mapping = config.getEsMapping();
    String sql = mapping.getSql();
    String condition = ESSyncUtil.pkConditionSql(mapping, data);
    sql = ESSyncUtil.appendCondition(sql, condition);
    DataSource ds = DatasourceConfig.DATA_SOURCES.get(config.getDataSourceKey());
    if (logger.isTraceEnabled()) {
        logger.trace("Main table delete es index by query sql, destination:{}, table: {}, index: {}, sql: {}", config.getDestination(), dml.getTable(), mapping.get_index(), sql.replace("\n", " "));
    }
    Util.sqlRS(ds, sql, rs -> {
        try {
            Map<String, Object> esFieldData = null;
            if (mapping.getPk() != null) {
                esFieldData = new LinkedHashMap<>();
                esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
                esFieldData.remove(mapping.getPk());
                for (String key : esFieldData.keySet()) {
                    esFieldData.put(Util.cleanColumn(key), null);
                }
            }
            while (rs.next()) {
                Object idVal = esTemplate.getIdValFromRS(mapping, rs);
                if (logger.isTraceEnabled()) {
                    logger.trace("Main table delete to es index by query sql, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), idVal);
                }
                esTemplate.delete(mapping, idVal, esFieldData);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return 0;
    });
}