use of io.mycat.sqlengine.mpp.ColumnRoutePair in project Mycat-Server by MyCATApache.
the class DruidSelectParser method changeSql.
/**
* 改写sql:需要加limit的加上
*/
@Override
public void changeSql(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt, LayerCachePool cachePool) throws SQLNonTransientException {
tryRoute(schema, rrs, cachePool);
rrs.copyLimitToNodes();
SQLSelectStatement selectStmt = (SQLSelectStatement) stmt;
SQLSelectQuery sqlSelectQuery = selectStmt.getSelect().getQuery();
if (sqlSelectQuery instanceof MySqlSelectQueryBlock) {
MySqlSelectQueryBlock mysqlSelectQuery = (MySqlSelectQueryBlock) selectStmt.getSelect().getQuery();
int limitStart = 0;
int limitSize = schema.getDefaultMaxLimit();
// clear group having
SQLSelectGroupByClause groupByClause = mysqlSelectQuery.getGroupBy();
// Modified by winbill, 20160614, do NOT include having clause when routing to multiple nodes
if (groupByClause != null && groupByClause.getHaving() != null && isRoutMultiNode(schema, rrs)) {
groupByClause.setHaving(null);
}
Map<String, Map<String, Set<ColumnRoutePair>>> allConditions = getAllConditions();
boolean isNeedAddLimit = isNeedAddLimit(schema, rrs, mysqlSelectQuery, allConditions);
if (isNeedAddLimit) {
Limit limit = new Limit();
limit.setRowCount(new SQLIntegerExpr(limitSize));
mysqlSelectQuery.setLimit(limit);
rrs.setLimitSize(limitSize);
String sql = getSql(rrs, stmt, isNeedAddLimit);
rrs.changeNodeSqlAfterAddLimit(schema, getCurentDbType(), sql, 0, limitSize, true);
}
Limit limit = mysqlSelectQuery.getLimit();
if (limit != null && !isNeedAddLimit) {
SQLIntegerExpr offset = (SQLIntegerExpr) limit.getOffset();
SQLIntegerExpr count = (SQLIntegerExpr) limit.getRowCount();
if (offset != null) {
limitStart = offset.getNumber().intValue();
rrs.setLimitStart(limitStart);
}
if (count != null) {
limitSize = count.getNumber().intValue();
rrs.setLimitSize(limitSize);
}
if (isNeedChangeLimit(rrs)) {
Limit changedLimit = new Limit();
changedLimit.setRowCount(new SQLIntegerExpr(limitStart + limitSize));
if (offset != null) {
if (limitStart < 0) {
String msg = "You have an error in your SQL syntax; check the manual that " + "corresponds to your MySQL server version for the right syntax to use near '" + limitStart + "'";
throw new SQLNonTransientException(ErrorCode.ER_PARSE_ERROR + " - " + msg);
} else {
changedLimit.setOffset(new SQLIntegerExpr(0));
}
}
mysqlSelectQuery.setLimit(changedLimit);
String sql = getSql(rrs, stmt, isNeedAddLimit);
rrs.changeNodeSqlAfterAddLimit(schema, getCurentDbType(), sql, 0, limitStart + limitSize, true);
// 设置改写后的sql
ctx.setSql(sql);
} else {
rrs.changeNodeSqlAfterAddLimit(schema, getCurentDbType(), getCtx().getSql(), rrs.getLimitStart(), rrs.getLimitSize(), true);
// ctx.setSql(nativeSql);
}
}
if (rrs.isDistTable()) {
SQLTableSource from = mysqlSelectQuery.getFrom();
for (RouteResultsetNode node : rrs.getNodes()) {
SQLIdentifierExpr sqlIdentifierExpr = new SQLIdentifierExpr();
sqlIdentifierExpr.setParent(from);
sqlIdentifierExpr.setName(node.getSubTableName());
SQLExprTableSource from2 = new SQLExprTableSource(sqlIdentifierExpr);
from2.setAlias(from.getAlias());
mysqlSelectQuery.setFrom(from2);
node.setStatement(stmt.toString());
}
}
rrs.setCacheAble(isNeedCache(schema, rrs, mysqlSelectQuery, allConditions));
}
}
use of io.mycat.sqlengine.mpp.ColumnRoutePair in project Mycat-Server by MyCATApache.
the class RouteCalculateUnit method addShardingExpr.
public void addShardingExpr(String tableName, String columnName, Object value) {
Map<String, Set<ColumnRoutePair>> tableColumnsMap = tablesAndConditions.get(tableName);
if (value == null) {
// where a=null
return;
}
if (tableColumnsMap == null) {
tableColumnsMap = new LinkedHashMap<String, Set<ColumnRoutePair>>();
tablesAndConditions.put(tableName, tableColumnsMap);
}
String uperColName = columnName.toUpperCase();
Set<ColumnRoutePair> columValues = tableColumnsMap.get(uperColName);
if (columValues == null) {
columValues = new LinkedHashSet<ColumnRoutePair>();
tablesAndConditions.get(tableName).put(uperColName, columValues);
}
if (value instanceof Object[]) {
for (Object item : (Object[]) value) {
if (item == null) {
continue;
}
columValues.add(new ColumnRoutePair(item.toString()));
}
} else if (value instanceof RangeValue) {
columValues.add(new ColumnRoutePair((RangeValue) value));
} else {
columValues.add(new ColumnRoutePair(value.toString()));
}
}
use of io.mycat.sqlengine.mpp.ColumnRoutePair in project Mycat-Server by MyCATApache.
the class RouterUtil method ruleCalculate.
/**
* @return dataNodeIndex -> [partitionKeysValueTuple+]
*/
public static Set<String> ruleCalculate(TableConfig tc, Set<ColumnRoutePair> colRoutePairSet, Map<String, Integer> dataNodeSlotMap) {
Set<String> routeNodeSet = new LinkedHashSet<String>();
String col = tc.getRule().getColumn();
RuleConfig rule = tc.getRule();
AbstractPartitionAlgorithm algorithm = rule.getRuleAlgorithm();
for (ColumnRoutePair colPair : colRoutePairSet) {
if (colPair.colValue != null) {
Integer nodeIndx = algorithm.calculate(colPair.colValue);
if (nodeIndx == null) {
throw new IllegalArgumentException("can't find datanode for sharding column:" + col + " val:" + colPair.colValue);
} else {
String dataNode = tc.getDataNodes().get(nodeIndx);
routeNodeSet.add(dataNode);
if (algorithm instanceof SlotFunction) {
dataNodeSlotMap.put(dataNode, ((SlotFunction) algorithm).slotValue());
}
colPair.setNodeId(nodeIndx);
}
} else if (colPair.rangeValue != null) {
Integer[] nodeRange = algorithm.calculateRange(String.valueOf(colPair.rangeValue.beginValue), String.valueOf(colPair.rangeValue.endValue));
if (nodeRange != null) {
/**
* 不能确认 colPair的 nodeid是否会有其它影响
*/
if (nodeRange.length == 0) {
routeNodeSet.addAll(tc.getDataNodes());
} else {
ArrayList<String> dataNodes = tc.getDataNodes();
String dataNode = null;
for (Integer nodeId : nodeRange) {
dataNode = dataNodes.get(nodeId);
if (algorithm instanceof SlotFunction) {
dataNodeSlotMap.put(dataNode, ((SlotFunction) algorithm).slotValue());
}
routeNodeSet.add(dataNode);
}
}
}
}
}
return routeNodeSet;
}
use of io.mycat.sqlengine.mpp.ColumnRoutePair in project Mycat-Server by MyCATApache.
the class RouterUtil method tryRouteForTables.
/**
* 多表路由
*/
public static RouteResultset tryRouteForTables(SchemaConfig schema, DruidShardingParseInfo ctx, RouteCalculateUnit routeUnit, RouteResultset rrs, boolean isSelect, LayerCachePool cachePool) throws SQLNonTransientException {
List<String> tables = ctx.getTables();
if (schema.isNoSharding() || (tables.size() >= 1 && isNoSharding(schema, tables.get(0)))) {
return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
}
// 只有一个表的
if (tables.size() == 1) {
return RouterUtil.tryRouteForOneTable(schema, ctx, routeUnit, tables.get(0), rrs, isSelect, cachePool);
}
Set<String> retNodesSet = new HashSet<String>();
// 每个表对应的路由映射
Map<String, Set<String>> tablesRouteMap = new HashMap<String, Set<String>>();
// 分库解析信息不为空
Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions = routeUnit.getTablesAndConditions();
if (tablesAndConditions != null && tablesAndConditions.size() > 0) {
// 为分库表找路由
RouterUtil.findRouteWithcConditionsForTables(schema, rrs, tablesAndConditions, tablesRouteMap, ctx.getSql(), cachePool, isSelect);
if (rrs.isFinishedRoute()) {
return rrs;
}
}
// 为全局表和单库表找路由
for (String tableName : tables) {
TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
if (tableConfig == null) {
// add 如果表读取不到则先将表名从别名中读取转化后再读取
String alias = ctx.getTableAliasMap().get(tableName);
if (!StringUtil.isEmpty(alias)) {
tableConfig = schema.getTables().get(alias.toUpperCase());
}
if (tableConfig == null) {
String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
}
if (tableConfig.isGlobalTable()) {
// 全局表
if (tablesRouteMap.get(tableName) == null) {
tablesRouteMap.put(tableName, new HashSet<String>());
}
tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
} else if (tablesRouteMap.get(tableName) == null) {
// 余下的表都是单库表
tablesRouteMap.put(tableName, new HashSet<String>());
tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
}
}
boolean isFirstAdd = true;
for (Map.Entry<String, Set<String>> entry : tablesRouteMap.entrySet()) {
if (entry.getValue() == null || entry.getValue().size() == 0) {
throw new SQLNonTransientException("parent key can't find any valid datanode ");
} else {
if (isFirstAdd) {
retNodesSet.addAll(entry.getValue());
isFirstAdd = false;
} else {
retNodesSet.retainAll(entry.getValue());
if (retNodesSet.size() == 0) {
// 两个表的路由无交集
String errMsg = "invalid route in sql, multi tables found but datanode has no intersection " + " sql:" + ctx.getSql();
LOGGER.warn(errMsg);
throw new SQLNonTransientException(errMsg);
}
}
}
}
if (retNodesSet != null && retNodesSet.size() > 0) {
String tableName = tables.get(0);
TableConfig tableConfig = schema.getTables().get(tableName.toUpperCase());
if (tableConfig.isDistTable()) {
routeToDistTableNode(tableName, schema, rrs, ctx.getSql(), tablesAndConditions, cachePool, isSelect);
return rrs;
}
if (retNodesSet.size() > 1 && isAllGlobalTable(ctx, schema)) {
// mulit routes ,not cache route result
if (isSelect) {
rrs.setCacheAble(false);
routeToSingleNode(rrs, retNodesSet.iterator().next(), ctx.getSql());
} else {
// delete 删除全局表的记录
routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql(), true);
}
} else {
routeToMultiNode(isSelect, rrs, retNodesSet, ctx.getSql());
}
}
return rrs;
}
use of io.mycat.sqlengine.mpp.ColumnRoutePair in project Mycat-Server by MyCATApache.
the class RouterUtil method routeToDistTableNode.
private static RouteResultset routeToDistTableNode(String tableName, SchemaConfig schema, RouteResultset rrs, String orgSql, Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions, LayerCachePool cachePool, boolean isSelect) throws SQLNonTransientException {
TableConfig tableConfig = schema.getTables().get(tableName);
if (tableConfig == null) {
String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
if (tableConfig.isGlobalTable()) {
String msg = "can't suport district table " + tableName + " schema:" + schema.getName() + " for global table ";
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
String partionCol = tableConfig.getPartitionColumn();
// String primaryKey = tableConfig.getPrimaryKey();
boolean isLoadData = false;
Set<String> tablesRouteSet = new HashSet<String>();
List<String> dataNodes = tableConfig.getDataNodes();
if (dataNodes.size() > 1) {
String msg = "can't suport district table " + tableName + " schema:" + schema.getName() + " for mutiple dataNode " + dataNodes;
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
String dataNode = dataNodes.get(0);
// 主键查找缓存暂时不实现
if (tablesAndConditions.isEmpty()) {
List<String> subTables = tableConfig.getDistTables();
tablesRouteSet.addAll(subTables);
}
for (Map.Entry<String, Map<String, Set<ColumnRoutePair>>> entry : tablesAndConditions.entrySet()) {
boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null;
Map<String, Set<ColumnRoutePair>> columnsMap = entry.getValue();
Set<ColumnRoutePair> partitionValue = columnsMap.get(partionCol);
if (partitionValue == null || partitionValue.size() == 0) {
tablesRouteSet.addAll(tableConfig.getDistTables());
} else {
for (ColumnRoutePair pair : partitionValue) {
AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
if (pair.colValue != null) {
Integer tableIndex = algorithm.calculate(pair.colValue);
if (tableIndex == null) {
String msg = "can't find any valid datanode :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
LOGGER.warn(msg);
throw new SQLNonTransientException(msg);
}
String subTable = tableConfig.getDistTables().get(tableIndex);
if (subTable != null) {
tablesRouteSet.add(subTable);
if (algorithm instanceof SlotFunction) {
rrs.getDataNodeSlotMap().put(subTable, ((SlotFunction) algorithm).slotValue());
}
}
}
if (pair.rangeValue != null) {
Integer[] tableIndexs = algorithm.calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
for (Integer idx : tableIndexs) {
String subTable = tableConfig.getDistTables().get(idx);
if (subTable != null) {
tablesRouteSet.add(subTable);
if (algorithm instanceof SlotFunction) {
rrs.getDataNodeSlotMap().put(subTable, ((SlotFunction) algorithm).slotValue());
}
}
}
}
}
}
}
Object[] subTables = tablesRouteSet.toArray();
RouteResultsetNode[] nodes = new RouteResultsetNode[subTables.length];
Map<String, Integer> dataNodeSlotMap = rrs.getDataNodeSlotMap();
for (int i = 0; i < nodes.length; i++) {
String table = String.valueOf(subTables[i]);
String changeSql = orgSql;
// rrs.getStatement()
nodes[i] = new RouteResultsetNode(dataNode, rrs.getSqlType(), changeSql);
nodes[i].setSubTableName(table);
nodes[i].setSource(rrs);
if (rrs.getDataNodeSlotMap().containsKey(dataNode)) {
nodes[i].setSlot(rrs.getDataNodeSlotMap().get(dataNode));
}
if (rrs.getCanRunInReadDB() != null) {
nodes[i].setCanRunInReadDB(rrs.getCanRunInReadDB());
}
if (dataNodeSlotMap.containsKey(table)) {
nodes[i].setSlot(dataNodeSlotMap.get(table));
}
if (rrs.getRunOnSlave() != null) {
nodes[0].setRunOnSlave(rrs.getRunOnSlave());
}
}
rrs.setNodes(nodes);
rrs.setSubTables(tablesRouteSet);
rrs.setFinishedRoute(true);
return rrs;
}
Aggregations