Examples with FieldItem com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem used on opensource projects

Example 16 with FieldItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem in project canal by alibaba.

the class ESSyncService method delete.

/**
 * 删除操作dml
 *
 * @param config es配置
 * @param dml dml数据
 */
private void delete(ESSyncConfig config, Dml dml) {
    List<Map<String, Object>> dataList = dml.getData();
    if (dataList == null || dataList.isEmpty()) {
        return;
    }
    SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
    for (Map<String, Object> data : dataList) {
        if (data == null || data.isEmpty()) {
            continue;
        }
        ESMapping mapping = config.getEsMapping();
        // ------是主表------
        if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
            if (mapping.get_id() != null) {
                FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping);
                // 主键为简单字段
                if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) {
                    Object idVal = esTemplate.getValFromData(mapping, data, idFieldItem.getFieldName(), idFieldItem.getColumn().getColumnName());
                    if (logger.isTraceEnabled()) {
                        logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), idVal);
                    }
                    esTemplate.delete(mapping, idVal, null);
                } else {
                    // ------主键带函数, 查询sql获取主键删除------
                    // FIXME 删除时反查sql为空记录, 无法获获取 id field 值
                    mainTableDelete(config, dml, data);
                }
            } else {
                FieldItem pkFieldItem = schemaItem.getIdFieldItem(mapping);
                if (!pkFieldItem.isMethod() && !pkFieldItem.isBinaryOp()) {
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    Object pkVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
                    if (logger.isTraceEnabled()) {
                        logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, pk: {}", config.getDestination(), dml.getTable(), mapping.get_index(), pkVal);
                    }
                    esFieldData.remove(pkFieldItem.getFieldName());
                    esFieldData.keySet().forEach(key -> esFieldData.put(key, null));
                    esTemplate.delete(mapping, pkVal, esFieldData);
                } else {
                    // ------主键带函数, 查询sql获取主键删除------
                    mainTableDelete(config, dml, data);
                }
            }
        }
        // 从表的操作
        for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
            if (tableItem.isMain()) {
                continue;
            }
            if (!tableItem.getTableName().equals(dml.getTable())) {
                continue;
            }
            // 关联条件出现在主表查询条件是否为简单字段
            boolean allFieldsSimple = true;
            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
                if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
                    allFieldsSimple = false;
                    break;
                }
            }
            // 所有查询字段均为简单字段
            if (allFieldsSimple) {
                // 不是子查询
                if (!tableItem.isSubQuery()) {
                    // ------关联表简单字段更新为null------
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
                        esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), null);
                    }
                    joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
                } else {
                    // ------关联子表简单字段更新------
                    subTableSimpleFieldOperation(config, dml, data, null, tableItem);
                }
            } else {
                // ------关联子表复杂字段更新 执行全sql更新es------
                wholeSqlOperation(config, dml, data, null, tableItem);
            }
        }
    }
}

Example 17 with FieldItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem in project canal by alibaba.

the class ES6xTemplate method getIdValFromRS.

@Override
public Object getIdValFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet) throws SQLException {
    SchemaItem schemaItem = mapping.getSchemaItem();
    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
    Object resultIdVal = null;
    for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
        Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
        if (fieldItem.getFieldName().equals(idFieldName)) {
            resultIdVal = value;
            break;
        }
    }
    return resultIdVal;
}

Example 18 with FieldItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem in project canal by alibaba.

the class ES6xTemplate method putRelationDataFromRS.

private void putRelationDataFromRS(ESMapping mapping, SchemaItem schemaItem, ResultSet resultSet, Map<String, Object> esFieldData) {
    // 添加父子文档关联信息
    if (!mapping.getRelations().isEmpty()) {
        mapping.getRelations().forEach((relationField, relationMapping) -> {
            Map<String, Object> relations = new HashMap<>();
            relations.put("name", relationMapping.getName());
            if (StringUtils.isNotEmpty(relationMapping.getParent())) {
                FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
                Object parentVal;
                try {
                    parentVal = getValFromRS(mapping, resultSet, parentFieldItem.getFieldName(), parentFieldItem.getFieldName());
                } catch (SQLException e) {
                    throw new RuntimeException(e);
                }
                if (parentVal != null) {
                    relations.put("parent", parentVal.toString());
                    esFieldData.put("$parent_routing", parentVal.toString());
                }
            }
            esFieldData.put(relationField, relations);
        });
    }
}

Example 19 with FieldItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem in project canal by alibaba.

the class ES6xTemplate method putRelationData.

private void putRelationData(ESMapping mapping, SchemaItem schemaItem, Map<String, Object> dmlData, Map<String, Object> esFieldData) {
    // 添加父子文档关联信息
    if (!mapping.getRelations().isEmpty()) {
        mapping.getRelations().forEach((relationField, relationMapping) -> {
            Map<String, Object> relations = new HashMap<>();
            relations.put("name", relationMapping.getName());
            if (StringUtils.isNotEmpty(relationMapping.getParent())) {
                FieldItem parentFieldItem = schemaItem.getSelectFields().get(relationMapping.getParent());
                String columnName = parentFieldItem.getColumnItems().iterator().next().getColumnName();
                Object parentVal = getValFromData(mapping, dmlData, parentFieldItem.getFieldName(), columnName);
                if (parentVal != null) {
                    relations.put("parent", parentVal.toString());
                    esFieldData.put("$parent_routing", parentVal.toString());
                }
            }
            esFieldData.put(relationField, relations);
        });
    }
}

Example 20 with FieldItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem in project canal by alibaba.

the class ES6xTemplate method getESDataFromDmlData.

@Override
public Object getESDataFromDmlData(ESSyncConfig.ESMapping mapping, Map<String, Object> dmlData, Map<String, Object> dmlOld, Map<String, Object> esFieldData) {
    SchemaItem schemaItem = mapping.getSchemaItem();
    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
    Object resultIdVal = null;
    for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
        String columnName = fieldItem.getColumnItems().iterator().next().getColumnName();
        if (fieldItem.getFieldName().equals(idFieldName)) {
            resultIdVal = getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName);
        }
        if (dmlOld.containsKey(columnName) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
            esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), getValFromData(mapping, dmlData, fieldItem.getFieldName(), columnName));
        }
    }
    // 添加父子文档关联信息
    putRelationData(mapping, schemaItem, dmlOld, esFieldData);
    return resultIdVal;
}