Examples with SchemaItem com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem used on opensource projects

Example 6 with SchemaItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem in project canal by alibaba.

the class ES7xTemplate method getESDataFromRS.

@Override
public Object getESDataFromRS(ESMapping mapping, ResultSet resultSet, Map<String, Object> dmlOld, Map<String, Object> esFieldData) throws SQLException {
    SchemaItem schemaItem = mapping.getSchemaItem();
    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
    Object resultIdVal = null;
    for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
        if (fieldItem.getFieldName().equals(idFieldName)) {
            resultIdVal = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
        }
        for (ColumnItem columnItem : fieldItem.getColumnItems()) {
            if (dmlOld.containsKey(columnItem.getColumnName()) && !mapping.getSkips().contains(fieldItem.getFieldName())) {
                esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName()));
                break;
            }
        }
    }
    // 添加父子文档关联信息
    putRelationDataFromRS(mapping, schemaItem, resultSet, esFieldData);
    return resultIdVal;
}

Example 7 with SchemaItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem in project canal by alibaba.

the class ES7xTemplate method getIdValFromRS.

@Override
public Object getIdValFromRS(ESMapping mapping, ResultSet resultSet) throws SQLException {
    SchemaItem schemaItem = mapping.getSchemaItem();
    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
    Object resultIdVal = null;
    for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
        Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
        if (fieldItem.getFieldName().equals(idFieldName)) {
            resultIdVal = value;
            break;
        }
    }
    return resultIdVal;
}

Example 8 with SchemaItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem in project canal by alibaba.

the class ESSyncUtil method pkConditionSql.

/**
 * 拼接主键条件
 *
 * @param mapping
 * @param data
 * @return
 */
public static String pkConditionSql(ESMapping mapping, Map<String, Object> data) {
    Set<ColumnItem> idColumns = new LinkedHashSet<>();
    SchemaItem schemaItem = mapping.getSchemaItem();
    TableItem mainTable = schemaItem.getMainTable();
    for (ColumnItem idColumnItem : schemaItem.getIdFieldItem(mapping).getColumnItems()) {
        if ((mainTable.getAlias() == null && idColumnItem.getOwner() == null) || (mainTable.getAlias() != null && mainTable.getAlias().equals(idColumnItem.getOwner()))) {
            idColumns.add(idColumnItem);
        }
    }
    if (idColumns.isEmpty()) {
        throw new RuntimeException("Not found primary key field in main table");
    }
    // 拼接condition
    StringBuilder condition = new StringBuilder(" ");
    for (ColumnItem idColumn : idColumns) {
        Object idVal = data.get(idColumn.getColumnName());
        if (mainTable.getAlias() != null)
            condition.append(mainTable.getAlias()).append(".");
        condition.append(idColumn.getColumnName()).append("=");
        if (idVal instanceof String) {
            condition.append("'").append(idVal).append("' AND ");
        } else {
            condition.append(idVal).append(" AND ");
        }
    }
    if (condition.toString().endsWith("AND ")) {
        int len2 = condition.length();
        condition.delete(len2 - 4, len2);
    }
    return condition.toString();
}

Example 9 with SchemaItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem in project canal by alibaba.

the class ESSyncService method delete.

/**
 * 删除操作dml
 *
 * @param config es配置
 * @param dml dml数据
 */
private void delete(ESSyncConfig config, Dml dml) {
    List<Map<String, Object>> dataList = dml.getData();
    if (dataList == null || dataList.isEmpty()) {
        return;
    }
    SchemaItem schemaItem = config.getEsMapping().getSchemaItem();
    for (Map<String, Object> data : dataList) {
        if (data == null || data.isEmpty()) {
            continue;
        }
        ESMapping mapping = config.getEsMapping();
        // ------是主表------
        if (schemaItem.getMainTable().getTableName().equalsIgnoreCase(dml.getTable())) {
            if (mapping.get_id() != null) {
                FieldItem idFieldItem = schemaItem.getIdFieldItem(mapping);
                // 主键为简单字段
                if (!idFieldItem.isMethod() && !idFieldItem.isBinaryOp()) {
                    Object idVal = esTemplate.getValFromData(mapping, data, idFieldItem.getFieldName(), idFieldItem.getColumn().getColumnName());
                    if (logger.isTraceEnabled()) {
                        logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, id: {}", config.getDestination(), dml.getTable(), mapping.get_index(), idVal);
                    }
                    esTemplate.delete(mapping, idVal, null);
                } else {
                    // ------主键带函数, 查询sql获取主键删除------
                    // FIXME 删除时反查sql为空记录, 无法获获取 id field 值
                    mainTableDelete(config, dml, data);
                }
            } else {
                FieldItem pkFieldItem = schemaItem.getIdFieldItem(mapping);
                if (!pkFieldItem.isMethod() && !pkFieldItem.isBinaryOp()) {
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    Object pkVal = esTemplate.getESDataFromDmlData(mapping, data, esFieldData);
                    if (logger.isTraceEnabled()) {
                        logger.trace("Main table delete es index, destination:{}, table: {}, index: {}, pk: {}", config.getDestination(), dml.getTable(), mapping.get_index(), pkVal);
                    }
                    esFieldData.remove(pkFieldItem.getFieldName());
                    esFieldData.keySet().forEach(key -> esFieldData.put(key, null));
                    esTemplate.delete(mapping, pkVal, esFieldData);
                } else {
                    // ------主键带函数, 查询sql获取主键删除------
                    mainTableDelete(config, dml, data);
                }
            }
        }
        // 从表的操作
        for (TableItem tableItem : schemaItem.getAliasTableItems().values()) {
            if (tableItem.isMain()) {
                continue;
            }
            if (!tableItem.getTableName().equals(dml.getTable())) {
                continue;
            }
            // 关联条件出现在主表查询条件是否为简单字段
            boolean allFieldsSimple = true;
            for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
                if (fieldItem.isMethod() || fieldItem.isBinaryOp()) {
                    allFieldsSimple = false;
                    break;
                }
            }
            // 所有查询字段均为简单字段
            if (allFieldsSimple) {
                // 不是子查询
                if (!tableItem.isSubQuery()) {
                    // ------关联表简单字段更新为null------
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    for (FieldItem fieldItem : tableItem.getRelationSelectFieldItems()) {
                        esFieldData.put(Util.cleanColumn(fieldItem.getFieldName()), null);
                    }
                    joinTableSimpleFieldOperation(config, dml, data, tableItem, esFieldData);
                } else {
                    // ------关联子表简单字段更新------
                    subTableSimpleFieldOperation(config, dml, data, null, tableItem);
                }
            } else {
                // ------关联子表复杂字段更新 执行全sql更新es------
                wholeSqlOperation(config, dml, data, null, tableItem);
            }
        }
    }
}

Example 10 with SchemaItem

use of com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem in project canal by alibaba.

the class ES6xTemplate method getIdValFromRS.

@Override
public Object getIdValFromRS(ESSyncConfig.ESMapping mapping, ResultSet resultSet) throws SQLException {
    SchemaItem schemaItem = mapping.getSchemaItem();
    String idFieldName = mapping.get_id() == null ? mapping.getPk() : mapping.get_id();
    Object resultIdVal = null;
    for (FieldItem fieldItem : schemaItem.getSelectFields().values()) {
        Object value = getValFromRS(mapping, resultSet, fieldItem.getFieldName(), fieldItem.getFieldName());
        if (fieldItem.getFieldName().equals(idFieldName)) {
            resultIdVal = value;
            break;
        }
    }
    return resultIdVal;
}