Examples with TableConfig io.mycat.config.model.TableConfig used on opensource projects

Example 16 with TableConfig

use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.

the class RouterUtil method getMetaReadDataNode.

/**
 * 根据表名随机获取一个节点
 *
 * @param schema     数据库名
 * @param table      表名
 * @return 			  数据节点
 * @author mycat
 */
private static String getMetaReadDataNode(SchemaConfig schema, String table) {
    // Table名字被转化为大写的，存储在schema
    table = table.toUpperCase();
    String dataNode = null;
    Map<String, TableConfig> tables = schema.getTables();
    TableConfig tc;
    if (tables != null && (tc = tables.get(table)) != null) {
        dataNode = getAliveRandomDataNode(tc);
    }
    return dataNode;
}

Example 17 with TableConfig

use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.

the class RouterUtil method tryRouteForTables.

/**
 * 多表路由
 */
public static RouteResultset tryRouteForTables(SchemaConfig schema, DruidShardingParseInfo ctx, RouteCalculateUnit routeUnit, RouteResultset rrs, boolean isSelect, LayerCachePool cachePool) throws SQLNonTransientException {
    List<String> tables = ctx.getTables();
    if (schema.isNoSharding() || (tables.size() >= 1 && isNoSharding(schema, tables.get(0)))) {
        return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
    }
    // 只有一个表的
    if (tables.size() == 1) {
        return RouterUtil.tryRouteForOneTable(schema, ctx, routeUnit, tables.get(0), rrs, isSelect, cachePool);
    }
    Set<String> retNodesSet = new HashSet<String>();
    // 每个表对应的路由映射
    Map<String, Set<String>> tablesRouteMap = new HashMap<String, Set<String>>();
    // 分库解析信息不为空
    Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions = routeUnit.getTablesAndConditions();
    if (tablesAndConditions != null && tablesAndConditions.size() > 0) {
        // 为分库表找路由
        RouterUtil.findRouteWithcConditionsForTables(schema, rrs, tablesAndConditions, tablesRouteMap, ctx.getSql(), cachePool, isSelect);
        if (rrs.isFinishedRoute()) {
            return rrs;
        }
    }
    // 为全局表和单库表找路由
    for (String tableName : tables) {
        TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
        if (tableConfig == null) {
            // add 如果表读取不到则先将表名从别名中读取转化后再读取
            String alias = ctx.getTableAliasMap().get(tableName);
            if (!StringUtil.isEmpty(alias)) {
                tableConfig = schema.getTables().get(alias.toUpperCase());
            }
            if (tableConfig == null) {
                String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
                LOGGER.warn(msg);
                throw new SQLNonTransientException(msg);
            }
        }
        if (tableConfig.isGlobalTable()) {
            // 全局表
            if (tablesRouteMap.get(tableName) == null) {
                tablesRouteMap.put(tableName, new HashSet<String>());
            }
            tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
        } else if (tablesRouteMap.get(tableName) == null) {
            // 余下的表都是单库表
            tablesRouteMap.put(tableName, new HashSet<String>());
            tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
        }
    }
    boolean isFirstAdd = true;
    for (Map.Entry<String, Set<String>> entry : tablesRouteMap.entrySet()) {
        if (entry.getValue() == null || entry.getValue().size() == 0) {
            throw new SQLNonTransientException("parent key can't find any valid datanode ");
        } else {
            if (isFirstAdd) {
                retNodesSet.addAll(entry.getValue());
                isFirstAdd = false;
            } else {
                retNodesSet.retainAll(entry.getValue());
                if (retNodesSet.size() == 0) {
                    // 两个表的路由无交集
                    String errMsg = "invalid route in sql, multi tables found but datanode has no intersection " + " sql:" + ctx.getSql();
                    LOGGER.warn(errMsg);
                    throw new SQLNonTransientException(errMsg);
                }
            }
        }
    }
    if (retNodesSet != null && retNodesSet.size() > 0) {
        String tableName = tables.get(0);
        TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
        if (tableConfig.isDistTable()) {
            routeToDistTableNode(tableName, schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect);
            return rrs;
        }
        if (retNodesSet.size() > 1 && isAllGlobalTable(ctx, schema)) {
            // mulit routes ,not cache route result
            if (isSelect) {
                rrs.setCacheAble(false);
                routeToSingleNode(rrs, retNodesSet.iterator().next(), ctx.getSql());
            } else {
                // delete 删除全局表的记录
                routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql(), true);
            }
        } else {
            routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql());
        }
    }
    return rrs;
}

Example 18 with TableConfig

use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.

the class RouterUtil method changeCreateTable.

private static String changeCreateTable(SchemaConfig schema, String tableName, String sql) {
    if (schema.getTables().containsKey(tableName)) {
        MySqlStatementParser parser = new MySqlStatementParser(sql);
        SQLStatement insertStatement = parser.parseStatement();
        if (insertStatement instanceof MySqlCreateTableStatement) {
            TableConfig tableConfig = schema.getTables().get(tableName);
            AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
            if (algorithm instanceof SlotFunction) {
                SQLColumnDefinition column = new SQLColumnDefinition();
                column.setDataType(new SQLCharacterDataType("int"));
                column.setName(new SQLIdentifierExpr("_slot"));
                column.setComment(new SQLCharExpr("自动迁移算法slot,禁止修改"));
                ((SQLCreateTableStatement) insertStatement).getTableElementList().add(column);
                return insertStatement.toString();
            }
        }
    }
    return sql;
}

Example 19 with TableConfig

use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.

the class RouterUtil method processERChildTable.

/**
 * 该方法，返回是否是ER子表
 * @param schema
 * @param origSQL
 * @param sc
 * @return
 * @throws SQLNonTransientException
 *
 * 备注说明：
 *     edit by ding.w at 2017.4.28, 主要处理 CLIENT_MULTI_STATEMENTS(insert into ; insert into)的情况
 *     目前仅支持mysql,并COM_QUERY请求包中的所有insert语句要么全部是er表，要么全部不是
 */
public static boolean processERChildTable(final SchemaConfig schema, final String origSQL, final ServerConnection sc) throws SQLNonTransientException {
    MySqlStatementParser parser = new MySqlStatementParser(origSQL);
    List<SQLStatement> statements = parser.parseStatementList();
    if (statements == null || statements.isEmpty()) {
        throw new SQLNonTransientException(String.format("无效的SQL语句:%s", origSQL));
    }
    // 是否是er表
    boolean erFlag = false;
    for (SQLStatement stmt : statements) {
        MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt;
        String tableName = insertStmt.getTableName().getSimpleName().toUpperCase();
        final TableConfig tc = schema.getTables().get(tableName);
        if (null != tc && tc.isChildTable()) {
            erFlag = true;
            String sql = insertStmt.toString();
            final RouteResultset rrs = new RouteResultset(sql, ServerParse.INSERT);
            String joinKey = tc.getJoinKey();
            // 因为是Insert语句，用MySqlInsertStatement进行parse
            // MySqlInsertStatement insertStmt = (MySqlInsertStatement) (new MySqlStatementParser(origSQL)).parseInsert();
            // 判断条件完整性，取得解析后语句列中的joinkey列的index
            int joinKeyIndex = getJoinKeyIndex(insertStmt.getColumns(), joinKey);
            if (joinKeyIndex == -1) {
                String inf = "joinKey not provided :" + tc.getJoinKey() + "," + insertStmt;
                LOGGER.warn(inf);
                throw new SQLNonTransientException(inf);
            }
            // 子表不支持批量插入
            if (isMultiInsert(insertStmt)) {
                String msg = "ChildTable multi insert not provided";
                LOGGER.warn(msg);
                throw new SQLNonTransientException(msg);
            }
            // 取得joinkey的值
            String joinKeyVal = insertStmt.getValues().getValues().get(joinKeyIndex).toString();
            // 解决bug #938，当关联字段的值为char类型时，去掉前后"'"
            String realVal = joinKeyVal;
            if (joinKeyVal.startsWith("'") && joinKeyVal.endsWith("'") && joinKeyVal.length() > 2) {
                realVal = joinKeyVal.substring(1, joinKeyVal.length() - 1);
            }
            // try to route by ER parent partion key
            // 如果是二级子表（父表不再有父表）,并且分片字段正好是joinkey字段，调用routeByERParentKey
            RouteResultset theRrs = RouterUtil.routeByERParentKey(sc, schema, ServerParse.INSERT, sql, rrs, tc, realVal);
            if (theRrs != null) {
                boolean processedInsert = false;
                // 判断是否需要全局序列号
                if (sc != null && tc.isAutoIncrement()) {
                    String primaryKey = tc.getPrimaryKey();
                    processedInsert = processInsert(sc, schema, ServerParse.INSERT, sql, tc.getName(), primaryKey);
                }
                if (processedInsert == false) {
                    rrs.setFinishedRoute(true);
                    sc.getSession2().execute(rrs, ServerParse.INSERT);
                }
                // 继续处理下一条
                continue;
            }
            // route by sql query root parent's datanode
            // 如果不是二级子表或者分片字段不是joinKey字段结果为空，则启动异步线程去后台分片查询出datanode
            // 只要查询出上一级表的parentkey字段的对应值在哪个分片即可
            final String findRootTBSql = tc.getLocateRTableKeySql().toLowerCase() + joinKeyVal;
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("find root parent's node sql " + findRootTBSql);
            }
            ListenableFuture<String> listenableFuture = MycatServer.getInstance().getListeningExecutorService().submit(new Callable<String>() {

                @Override
                public String call() throws Exception {
                    FetchStoreNodeOfChildTableHandler fetchHandler = new FetchStoreNodeOfChildTableHandler();
                    // return fetchHandler.execute(schema.getName(), findRootTBSql, tc.getRootParent().getDataNodes());
                    return fetchHandler.execute(schema.getName(), findRootTBSql, tc.getRootParent().getDataNodes(), sc);
                }
            });
            Futures.addCallback(listenableFuture, new FutureCallback<String>() {

                @Override
                public void onSuccess(String result) {
                    // 结果为空，证明上一级表中不存在那条记录，失败
                    if (Strings.isNullOrEmpty(result)) {
                        StringBuilder s = new StringBuilder();
                        LOGGER.warn(s.append(sc.getSession2()).append(origSQL).toString() + " err:" + "can't find (root) parent sharding node for sql:" + origSQL);
                        if (!sc.isAutocommit()) {
                            // 处于事务下失败, 必须回滚
                            sc.setTxInterrupt("can't find (root) parent sharding node for sql:" + origSQL);
                        }
                        sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "can't find (root) parent sharding node for sql:" + origSQL);
                        return;
                    }
                    if (LOGGER.isDebugEnabled()) {
                        LOGGER.debug("found partion node for child table to insert " + result + " sql :" + origSQL);
                    }
                    // 找到分片，进行插入（和其他的一样，需要判断是否需要全局自增ID）
                    boolean processedInsert = false;
                    if (sc != null && tc.isAutoIncrement()) {
                        try {
                            String primaryKey = tc.getPrimaryKey();
                            processedInsert = processInsert(sc, schema, ServerParse.INSERT, origSQL, tc.getName(), primaryKey);
                        } catch (SQLNonTransientException e) {
                            LOGGER.warn("sequence processInsert error,", e);
                            sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "sequence processInsert error," + e.getMessage());
                        }
                    }
                    if (processedInsert == false) {
                        RouteResultset executeRrs = RouterUtil.routeToSingleNode(rrs, result, origSQL);
                        sc.getSession2().execute(executeRrs, ServerParse.INSERT);
                    }
                }

                @Override
                public void onFailure(Throwable t) {
                    StringBuilder s = new StringBuilder();
                    LOGGER.warn(s.append(sc.getSession2()).append(origSQL).toString() + " err:" + t.getMessage());
                    sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, t.getMessage() + " " + s.toString());
                }
            }, MycatServer.getInstance().getListeningExecutorService());
        } else if (erFlag) {
            throw new SQLNonTransientException(String.format("%s包含不是ER分片的表", origSQL));
        }
    }
    return erFlag;
}

Example 20 with TableConfig

use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.

the class RouterUtil method findRouteWithcConditionsForTables.

/**
 * 处理分库表路由
 */
public static void findRouteWithcConditionsForTables(SchemaConfig schema, RouteResultset rrs, Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions, Map<String, Set<String>> tablesRouteMap, String sql, LayerCachePool cachePool, boolean isSelect) throws SQLNonTransientException {
    // 为分库表找路由
    for (Map.Entry<String, Map<String, Set<ColumnRoutePair>>> entry : tablesAndConditions.entrySet()) {
        String tableName = entry.getKey().toUpperCase();
        TableConfig tableConfig = schema.getTables().get(tableName);
        if (tableConfig == null) {
            String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
            LOGGER.warn(msg);
            throw new SQLNonTransientException(msg);
        }
        if (tableConfig.getDistTables() != null && tableConfig.getDistTables().size() > 0) {
            routeToDistTableNode(tableName, schema, rrs, sql, tablesAndConditions, cachePool, isSelect);
        }
        // 全局表或者不分库的表略过（全局表后面再计算）
        if (tableConfig.isGlobalTable() || schema.getTables().get(tableName).getDataNodes().size() == 1) {
            continue;
        } else {
            // 非全局表：分库表、childTable、其他
            Map<String, Set<ColumnRoutePair>> columnsMap = entry.getValue();
            String joinKey = tableConfig.getJoinKey();
            String partionCol = tableConfig.getPartitionColumn();
            String primaryKey = tableConfig.getPrimaryKey();
            boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null;
            boolean isLoadData = false;
            if (LOGGER.isDebugEnabled() && sql.startsWith(LoadData.loadDataHint) || rrs.isLoadData()) {
                // 由于load data一次会计算很多路由数据，如果输出此日志会极大降低load data的性能
                isLoadData = true;
            }
            if (entry.getValue().get(primaryKey) != null && entry.getValue().size() == 1 && !isLoadData) {
                // 主键查找
                // try by primary key if found in cache
                Set<ColumnRoutePair> primaryKeyPairs = entry.getValue().get(primaryKey);
                if (primaryKeyPairs != null) {
                    if (LOGGER.isDebugEnabled()) {
                        LOGGER.debug("try to find cache by primary key ");
                    }
                    String tableKey = schema.getName() + '_' + tableName;
                    boolean allFound = true;
                    for (ColumnRoutePair pair : primaryKeyPairs) {
                        // 可能id in(1,2,3)多主键
                        String cacheKey = pair.colValue;
                        String dataNode = (String) cachePool.get(tableKey, cacheKey);
                        if (dataNode == null) {
                            allFound = false;
                            continue;
                        } else {
                            if (tablesRouteMap.get(tableName) == null) {
                                tablesRouteMap.put(tableName, new HashSet<String>());
                            }
                            tablesRouteMap.get(tableName).add(dataNode);
                            continue;
                        }
                    }
                    if (!allFound) {
                        // need cache primary key ->datanode relation
                        if (isSelect && tableConfig.getPrimaryKey() != null) {
                            rrs.setPrimaryKey(tableKey + '.' + tableConfig.getPrimaryKey());
                        }
                    } else {
                        // 主键缓存中找到了就执行循环的下一轮
                        continue;
                    }
                }
            }
            if (isFoundPartitionValue) {
                // 分库表
                Set<ColumnRoutePair> partitionValue = columnsMap.get(partionCol);
                if (partitionValue == null || partitionValue.size() == 0) {
                    if (tablesRouteMap.get(tableName) == null) {
                        tablesRouteMap.put(tableName, new HashSet<String>());
                    }
                    tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
                } else {
                    for (ColumnRoutePair pair : partitionValue) {
                        AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
                        if (pair.colValue != null) {
                            Integer nodeIndex = algorithm.calculate(pair.colValue);
                            if (nodeIndex == null) {
                                String msg = "can't find any valid datanode :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
                                LOGGER.warn(msg);
                                throw new SQLNonTransientException(msg);
                            }
                            ArrayList<String> dataNodes = tableConfig.getDataNodes();
                            String node;
                            if (nodeIndex >= 0 && nodeIndex < dataNodes.size()) {
                                node = dataNodes.get(nodeIndex);
                            } else {
                                node = null;
                                String msg = "Can't find a valid data node for specified node index :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue + " -> " + "Index : " + nodeIndex;
                                LOGGER.warn(msg);
                                throw new SQLNonTransientException(msg);
                            }
                            if (node != null) {
                                if (tablesRouteMap.get(tableName) == null) {
                                    tablesRouteMap.put(tableName, new HashSet<String>());
                                }
                                if (algorithm instanceof SlotFunction) {
                                    rrs.getDataNodeSlotMap().put(node, ((SlotFunction) algorithm).slotValue());
                                }
                                tablesRouteMap.get(tableName).add(node);
                            }
                        }
                        if (pair.rangeValue != null) {
                            Integer[] nodeIndexs = algorithm.calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
                            ArrayList<String> dataNodes = tableConfig.getDataNodes();
                            String node;
                            for (Integer idx : nodeIndexs) {
                                if (idx >= 0 && idx < dataNodes.size()) {
                                    node = dataNodes.get(idx);
                                } else {
                                    String msg = "Can't find valid data node(s) for some of specified node indexes :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn();
                                    LOGGER.warn(msg);
                                    throw new SQLNonTransientException(msg);
                                }
                                if (node != null) {
                                    if (tablesRouteMap.get(tableName) == null) {
                                        tablesRouteMap.put(tableName, new HashSet<String>());
                                    }
                                    if (algorithm instanceof SlotFunction) {
                                        rrs.getDataNodeSlotMap().put(node, ((SlotFunction) algorithm).slotValue());
                                    }
                                    tablesRouteMap.get(tableName).add(node);
                                }
                            }
                        }
                    }
                }
            } else if (joinKey != null && columnsMap.get(joinKey) != null && columnsMap.get(joinKey).size() != 0) {
                // childTable  (如果是select 语句的父子表join)之前要找到root table,将childTable移除,只留下root table
                Set<ColumnRoutePair> joinKeyValue = columnsMap.get(joinKey);
                Set<String> dataNodeSet = ruleByJoinValueCalculate(rrs, tableConfig, joinKeyValue);
                if (dataNodeSet.isEmpty()) {
                    throw new SQLNonTransientException("parent key can't find any valid datanode ");
                }
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("found partion nodes (using parent partion rule directly) for child table to update  " + Arrays.toString(dataNodeSet.toArray()) + " sql :" + sql);
                }
                if (dataNodeSet.size() > 1) {
                    routeToMultiNode(rrs.isCacheAble(), rrs, dataNodeSet, sql);
                    rrs.setFinishedRoute(true);
                    return;
                } else {
                    rrs.setCacheAble(true);
                    routeToSingleNode(rrs, dataNodeSet.iterator().next(), sql);
                    return;
                }
            } else {
                // 没找到拆分字段，该表的所有节点都路由
                if (tablesRouteMap.get(tableName) == null) {
                    tablesRouteMap.put(tableName, new HashSet<String>());
                }
                boolean isSlotFunction = tableConfig.getRule() != null && tableConfig.getRule().getRuleAlgorithm() instanceof SlotFunction;
                if (isSlotFunction) {
                    for (String dn : tableConfig.getDataNodes()) {
                        rrs.getDataNodeSlotMap().put(dn, -1);
                    }
                }
                tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
            }
        }
    }
}