use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.
the class RouterUtil method getMetaReadDataNode.
/**
* 根据表名随机获取一个节点
*
* @param schema 数据库名
* @param table 表名
* @return 数据节点
* @author mycat
*/
private static String getMetaReadDataNode(SchemaConfig schema, String table) {
// Table名字被转化为大写的,存储在schema
table = table.toUpperCase();
String dataNode = null;
Map<String, TableConfig> tables = schema.getTables();
TableConfig tc;
if (tables != null && (tc = tables.get(table)) != null) {
dataNode = getAliveRandomDataNode(tc);
}
return dataNode;
}
use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.
the class RouterUtil method tryRouteForTables.
/**
* 多表路由
*/
public static RouteResultset tryRouteForTables(SchemaConfig schema, DruidShardingParseInfo ctx, RouteCalculateUnit routeUnit, RouteResultset rrs, boolean isSelect, LayerCachePool cachePool) throws SQLNonTransientException {
List<String> tables = ctx.getTables();
if (schema.isNoSharding() || (tables.size() >= 1 && isNoSharding(schema, tables.get(0)))) {
return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
}
// 只有一个表的
if (tables.size() == 1) {
return RouterUtil.tryRouteForOneTable(schema, ctx, routeUnit, tables.get(0), rrs, isSelect, cachePool);
}
Set<String> retNodesSet = new HashSet<String>();
// 每个表对应的路由映射
Map<String, Set<String>> tablesRouteMap = new HashMap<String, Set<String>>();
// 分库解析信息不为空
Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions = routeUnit.getTablesAndConditions();
if (tablesAndConditions != null && tablesAndConditions.size() > 0) {
// 为分库表找路由
RouterUtil.findRouteWithcConditionsForTables(schema, rrs, tablesAndConditions, tablesRouteMap, ctx.getSql(), cachePool, isSelect);
if (rrs.isFinishedRoute()) {
return rrs;
}
}
// 为全局表和单库表找路由
for (String tableName : tables) {
TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
if (tableConfig == null) {
// add 如果表读取不到则先将表名从别名中读取转化后再读取
String alias = ctx.getTableAliasMap().get(tableName);
if (!StringUtil.isEmpty(alias)) {
tableConfig = schema.getTables().get(alias.toUpperCase());
}
if (tableConfig == null) {
String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
}
if (tableConfig.isGlobalTable()) {
// 全局表
if (tablesRouteMap.get(tableName) == null) {
tablesRouteMap.put(tableName, new HashSet<String>());
}
tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
} else if (tablesRouteMap.get(tableName) == null) {
// 余下的表都是单库表
tablesRouteMap.put(tableName, new HashSet<String>());
tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
}
}
boolean isFirstAdd = true;
for (Map.Entry<String, Set<String>> entry : tablesRouteMap.entrySet()) {
if (entry.getValue() == null || entry.getValue().size() == 0) {
throw new SQLNonTransientException("parent key can't find any valid datanode ");
} else {
if (isFirstAdd) {
retNodesSet.addAll(entry.getValue());
isFirstAdd = false;
} else {
retNodesSet.retainAll(entry.getValue());
if (retNodesSet.size() == 0) {
// 两个表的路由无交集
String errMsg = "invalid route in sql, multi tables found but datanode has no intersection " + " sql:" + ctx.getSql();
LOGGER.warn(errMsg);
throw new SQLNonTransientException(errMsg);
}
}
}
}
if (retNodesSet != null && retNodesSet.size() > 0) {
String tableName = tables.get(0);
TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
if (tableConfig.isDistTable()) {
routeToDistTableNode(tableName, schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect);
return rrs;
}
if (retNodesSet.size() > 1 && isAllGlobalTable(ctx, schema)) {
// mulit routes ,not cache route result
if (isSelect) {
rrs.setCacheAble(false);
routeToSingleNode(rrs, retNodesSet.iterator().next(), ctx.getSql());
} else {
// delete 删除全局表的记录
routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql(), true);
}
} else {
routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql());
}
}
return rrs;
}
use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.
the class RouterUtil method changeCreateTable.
private static String changeCreateTable(SchemaConfig schema, String tableName, String sql) {
if (schema.getTables().containsKey(tableName)) {
MySqlStatementParser parser = new MySqlStatementParser(sql);
SQLStatement insertStatement = parser.parseStatement();
if (insertStatement instanceof MySqlCreateTableStatement) {
TableConfig tableConfig = schema.getTables().get(tableName);
AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
if (algorithm instanceof SlotFunction) {
SQLColumnDefinition column = new SQLColumnDefinition();
column.setDataType(new SQLCharacterDataType("int"));
column.setName(new SQLIdentifierExpr("_slot"));
column.setComment(new SQLCharExpr("自动迁移算法slot,禁止修改"));
((SQLCreateTableStatement) insertStatement).getTableElementList().add(column);
return insertStatement.toString();
}
}
}
return sql;
}
use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.
the class RouterUtil method processERChildTable.
/**
* 该方法,返回是否是ER子表
* @param schema
* @param origSQL
* @param sc
* @return
* @throws SQLNonTransientException
*
* 备注说明:
* edit by ding.w at 2017.4.28, 主要处理 CLIENT_MULTI_STATEMENTS(insert into ; insert into)的情况
* 目前仅支持mysql,并COM_QUERY请求包中的所有insert语句要么全部是er表,要么全部不是
*/
public static boolean processERChildTable(final SchemaConfig schema, final String origSQL, final ServerConnection sc) throws SQLNonTransientException {
MySqlStatementParser parser = new MySqlStatementParser(origSQL);
List<SQLStatement> statements = parser.parseStatementList();
if (statements == null || statements.isEmpty()) {
throw new SQLNonTransientException(String.format("无效的SQL语句:%s", origSQL));
}
// 是否是er表
boolean erFlag = false;
for (SQLStatement stmt : statements) {
MySqlInsertStatement insertStmt = (MySqlInsertStatement) stmt;
String tableName = insertStmt.getTableName().getSimpleName().toUpperCase();
final TableConfig tc = schema.getTables().get(tableName);
if (null != tc && tc.isChildTable()) {
erFlag = true;
String sql = insertStmt.toString();
final RouteResultset rrs = new RouteResultset(sql, ServerParse.INSERT);
String joinKey = tc.getJoinKey();
// 因为是Insert语句,用MySqlInsertStatement进行parse
// MySqlInsertStatement insertStmt = (MySqlInsertStatement) (new MySqlStatementParser(origSQL)).parseInsert();
// 判断条件完整性,取得解析后语句列中的joinkey列的index
int joinKeyIndex = getJoinKeyIndex(insertStmt.getColumns(), joinKey);
if (joinKeyIndex == -1) {
String inf = "joinKey not provided :" + tc.getJoinKey() + "," + insertStmt;
LOGGER.warn(inf);
throw new SQLNonTransientException(inf);
}
// 子表不支持批量插入
if (isMultiInsert(insertStmt)) {
String msg = "ChildTable multi insert not provided";
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
// 取得joinkey的值
String joinKeyVal = insertStmt.getValues().getValues().get(joinKeyIndex).toString();
// 解决bug #938,当关联字段的值为char类型时,去掉前后"'"
String realVal = joinKeyVal;
if (joinKeyVal.startsWith("'") && joinKeyVal.endsWith("'") && joinKeyVal.length() > 2) {
realVal = joinKeyVal.substring(1, joinKeyVal.length() - 1);
}
// try to route by ER parent partion key
// 如果是二级子表(父表不再有父表),并且分片字段正好是joinkey字段,调用routeByERParentKey
RouteResultset theRrs = RouterUtil.routeByERParentKey(sc, schema, ServerParse.INSERT, sql, rrs, tc, realVal);
if (theRrs != null) {
boolean processedInsert = false;
// 判断是否需要全局序列号
if (sc != null && tc.isAutoIncrement()) {
String primaryKey = tc.getPrimaryKey();
processedInsert = processInsert(sc, schema, ServerParse.INSERT, sql, tc.getName(), primaryKey);
}
if (processedInsert == false) {
rrs.setFinishedRoute(true);
sc.getSession2().execute(rrs, ServerParse.INSERT);
}
// 继续处理下一条
continue;
}
// route by sql query root parent's datanode
// 如果不是二级子表或者分片字段不是joinKey字段结果为空,则启动异步线程去后台分片查询出datanode
// 只要查询出上一级表的parentkey字段的对应值在哪个分片即可
final String findRootTBSql = tc.getLocateRTableKeySql().toLowerCase() + joinKeyVal;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("find root parent's node sql " + findRootTBSql);
}
ListenableFuture<String> listenableFuture = MycatServer.getInstance().getListeningExecutorService().submit(new Callable<String>() {
@Override
public String call() throws Exception {
FetchStoreNodeOfChildTableHandler fetchHandler = new FetchStoreNodeOfChildTableHandler();
// return fetchHandler.execute(schema.getName(), findRootTBSql, tc.getRootParent().getDataNodes());
return fetchHandler.execute(schema.getName(), findRootTBSql, tc.getRootParent().getDataNodes(), sc);
}
});
Futures.addCallback(listenableFuture, new FutureCallback<String>() {
@Override
public void onSuccess(String result) {
// 结果为空,证明上一级表中不存在那条记录,失败
if (Strings.isNullOrEmpty(result)) {
StringBuilder s = new StringBuilder();
LOGGER.warn(s.append(sc.getSession2()).append(origSQL).toString() + " err:" + "can't find (root) parent sharding node for sql:" + origSQL);
if (!sc.isAutocommit()) {
// 处于事务下失败, 必须回滚
sc.setTxInterrupt("can't find (root) parent sharding node for sql:" + origSQL);
}
sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "can't find (root) parent sharding node for sql:" + origSQL);
return;
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("found partion node for child table to insert " + result + " sql :" + origSQL);
}
// 找到分片,进行插入(和其他的一样,需要判断是否需要全局自增ID)
boolean processedInsert = false;
if (sc != null && tc.isAutoIncrement()) {
try {
String primaryKey = tc.getPrimaryKey();
processedInsert = processInsert(sc, schema, ServerParse.INSERT, origSQL, tc.getName(), primaryKey);
} catch (SQLNonTransientException e) {
LOGGER.warn("sequence processInsert error,", e);
sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, "sequence processInsert error," + e.getMessage());
}
}
if (processedInsert == false) {
RouteResultset executeRrs = RouterUtil.routeToSingleNode(rrs, result, origSQL);
sc.getSession2().execute(executeRrs, ServerParse.INSERT);
}
}
@Override
public void onFailure(Throwable t) {
StringBuilder s = new StringBuilder();
LOGGER.warn(s.append(sc.getSession2()).append(origSQL).toString() + " err:" + t.getMessage());
sc.writeErrMessage(ErrorCode.ER_PARSE_ERROR, t.getMessage() + " " + s.toString());
}
}, MycatServer.getInstance().getListeningExecutorService());
} else if (erFlag) {
throw new SQLNonTransientException(String.format("%s包含不是ER分片的表", origSQL));
}
}
return erFlag;
}
use of io.mycat.config.model.TableConfig in project Mycat_plus by coderczp.
the class RouterUtil method findRouteWithcConditionsForTables.
/**
* 处理分库表路由
*/
public static void findRouteWithcConditionsForTables(SchemaConfig schema, RouteResultset rrs, Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions, Map<String, Set<String>> tablesRouteMap, String sql, LayerCachePool cachePool, boolean isSelect) throws SQLNonTransientException {
// 为分库表找路由
for (Map.Entry<String, Map<String, Set<ColumnRoutePair>>> entry : tablesAndConditions.entrySet()) {
String tableName = entry.getKey().toUpperCase();
TableConfig tableConfig = schema.getTables().get(tableName);
if (tableConfig == null) {
String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
if (tableConfig.getDistTables() != null && tableConfig.getDistTables().size() > 0) {
routeToDistTableNode(tableName, schema, rrs, sql, tablesAndConditions, cachePool, isSelect);
}
// 全局表或者不分库的表略过(全局表后面再计算)
if (tableConfig.isGlobalTable() || schema.getTables().get(tableName).getDataNodes().size() == 1) {
continue;
} else {
// 非全局表:分库表、childTable、其他
Map<String, Set<ColumnRoutePair>> columnsMap = entry.getValue();
String joinKey = tableConfig.getJoinKey();
String partionCol = tableConfig.getPartitionColumn();
String primaryKey = tableConfig.getPrimaryKey();
boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null;
boolean isLoadData = false;
if (LOGGER.isDebugEnabled() && sql.startsWith(LoadData.loadDataHint) || rrs.isLoadData()) {
// 由于load data一次会计算很多路由数据,如果输出此日志会极大降低load data的性能
isLoadData = true;
}
if (entry.getValue().get(primaryKey) != null && entry.getValue().size() == 1 && !isLoadData) {
// 主键查找
// try by primary key if found in cache
Set<ColumnRoutePair> primaryKeyPairs = entry.getValue().get(primaryKey);
if (primaryKeyPairs != null) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("try to find cache by primary key ");
}
String tableKey = schema.getName() + '_' + tableName;
boolean allFound = true;
for (ColumnRoutePair pair : primaryKeyPairs) {
// 可能id in(1,2,3)多主键
String cacheKey = pair.colValue;
String dataNode = (String) cachePool.get(tableKey, cacheKey);
if (dataNode == null) {
allFound = false;
continue;
} else {
if (tablesRouteMap.get(tableName) == null) {
tablesRouteMap.put(tableName, new HashSet<String>());
}
tablesRouteMap.get(tableName).add(dataNode);
continue;
}
}
if (!allFound) {
// need cache primary key ->datanode relation
if (isSelect && tableConfig.getPrimaryKey() != null) {
rrs.setPrimaryKey(tableKey + '.' + tableConfig.getPrimaryKey());
}
} else {
// 主键缓存中找到了就执行循环的下一轮
continue;
}
}
}
if (isFoundPartitionValue) {
// 分库表
Set<ColumnRoutePair> partitionValue = columnsMap.get(partionCol);
if (partitionValue == null || partitionValue.size() == 0) {
if (tablesRouteMap.get(tableName) == null) {
tablesRouteMap.put(tableName, new HashSet<String>());
}
tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
} else {
for (ColumnRoutePair pair : partitionValue) {
AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
if (pair.colValue != null) {
Integer nodeIndex = algorithm.calculate(pair.colValue);
if (nodeIndex == null) {
String msg = "can't find any valid datanode :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
ArrayList<String> dataNodes = tableConfig.getDataNodes();
String node;
if (nodeIndex >= 0 && nodeIndex < dataNodes.size()) {
node = dataNodes.get(nodeIndex);
} else {
node = null;
String msg = "Can't find a valid data node for specified node index :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue + " -> " + "Index : " + nodeIndex;
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
if (node != null) {
if (tablesRouteMap.get(tableName) == null) {
tablesRouteMap.put(tableName, new HashSet<String>());
}
if (algorithm instanceof SlotFunction) {
rrs.getDataNodeSlotMap().put(node, ((SlotFunction) algorithm).slotValue());
}
tablesRouteMap.get(tableName).add(node);
}
}
if (pair.rangeValue != null) {
Integer[] nodeIndexs = algorithm.calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
ArrayList<String> dataNodes = tableConfig.getDataNodes();
String node;
for (Integer idx : nodeIndexs) {
if (idx >= 0 && idx < dataNodes.size()) {
node = dataNodes.get(idx);
} else {
String msg = "Can't find valid data node(s) for some of specified node indexes :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn();
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
if (node != null) {
if (tablesRouteMap.get(tableName) == null) {
tablesRouteMap.put(tableName, new HashSet<String>());
}
if (algorithm instanceof SlotFunction) {
rrs.getDataNodeSlotMap().put(node, ((SlotFunction) algorithm).slotValue());
}
tablesRouteMap.get(tableName).add(node);
}
}
}
}
}
} else if (joinKey != null && columnsMap.get(joinKey) != null && columnsMap.get(joinKey).size() != 0) {
// childTable (如果是select 语句的父子表join)之前要找到root table,将childTable移除,只留下root table
Set<ColumnRoutePair> joinKeyValue = columnsMap.get(joinKey);
Set<String> dataNodeSet = ruleByJoinValueCalculate(rrs, tableConfig, joinKeyValue);
if (dataNodeSet.isEmpty()) {
throw new SQLNonTransientException("parent key can't find any valid datanode ");
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("found partion nodes (using parent partion rule directly) for child table to update " + Arrays.toString(dataNodeSet.toArray()) + " sql :" + sql);
}
if (dataNodeSet.size() > 1) {
routeToMultiNode(rrs.isCacheAble(), rrs, dataNodeSet, sql);
rrs.setFinishedRoute(true);
return;
} else {
rrs.setCacheAble(true);
routeToSingleNode(rrs, dataNodeSet.iterator().next(), sql);
return;
}
} else {
// 没找到拆分字段,该表的所有节点都路由
if (tablesRouteMap.get(tableName) == null) {
tablesRouteMap.put(tableName, new HashSet<String>());
}
boolean isSlotFunction = tableConfig.getRule() != null && tableConfig.getRule().getRuleAlgorithm() instanceof SlotFunction;
if (isSlotFunction) {
for (String dn : tableConfig.getDataNodes()) {
rrs.getDataNodeSlotMap().put(dn, -1);
}
}
tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
}
}
}
}
Aggregations