Examples with AbstractPartitionAlgorithm io.mycat.route.function.AbstractPartitionAlgorithm used on opensource projects

Example 6 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class RuleDataPathChildrenCacheListener method reloadRuleData.

private void reloadRuleData(String name) {
    String tableName = name.substring(name.lastIndexOf("_") + 1, name.indexOf("."));
    String ruleName = name.substring(0, name.indexOf("."));
    Map<String, SchemaConfig> schemaConfigMap = MycatServer.getInstance().getConfig().getSchemas();
    for (SchemaConfig schemaConfig : schemaConfigMap.values()) {
        TableConfig tableConfig = schemaConfig.getTables().get(tableName.toUpperCase());
        if (tableConfig == null)
            continue;
        RuleConfig rule = tableConfig.getRule();
        AbstractPartitionAlgorithm function = rule.getRuleAlgorithm();
        if (function instanceof ReloadFunction) {
            ((ReloadFunction) function).reload();
        }
    }
}

Example 7 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class DruidCreateTableParser method statementParse.

@Override
public void statementParse(SchemaConfig schema, RouteResultset rrs, SQLStatement stmt) throws SQLNonTransientException {
    MySqlCreateTableStatement createStmt = (MySqlCreateTableStatement) stmt;
    if (createStmt.getQuery() != null) {
        String msg = "create table from other table not supported :" + stmt;
        LOGGER.warn(msg);
        throw new SQLNonTransientException(msg);
    }
    String tableName = StringUtil.removeBackquote(createStmt.getTableSource().toString().toUpperCase());
    if (schema.getTables().containsKey(tableName)) {
        TableConfig tableConfig = schema.getTables().get(tableName);
        AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
        if (algorithm instanceof SlotFunction) {
            SQLColumnDefinition column = new SQLColumnDefinition();
            column.setDataType(new SQLCharacterDataType("int"));
            column.setName(new SQLIdentifierExpr("_slot"));
            column.setComment(new SQLCharExpr("自动迁移算法slot,禁止修改"));
            ((SQLCreateTableStatement) stmt).getTableElementList().add(column);
            String sql = createStmt.toString();
            rrs.setStatement(sql);
            ctx.setSql(sql);
        }
    }
    ctx.addTable(tableName);
}

Example 8 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class DruidInsertParser method parserBatchInsert.

/**
	 * insert into .... select .... 或insert into table() values (),(),....
	 * @param schema
	 * @param rrs
	 * @param insertStmt
	 * @throws SQLNonTransientException
	 */
private void parserBatchInsert(SchemaConfig schema, RouteResultset rrs, String partitionColumn, String tableName, MySqlInsertStatement insertStmt) throws SQLNonTransientException {
    //insert into table() values (),(),....
    if (insertStmt.getValuesList().size() > 1) {
        //字段列数
        int columnNum = insertStmt.getColumns().size();
        int shardingColIndex = getShardingColIndex(insertStmt, partitionColumn);
        if (shardingColIndex == -1) {
            String msg = "bad insert sql (sharding column:" + partitionColumn + " not provided," + insertStmt;
            LOGGER.warn(msg);
            throw new SQLNonTransientException(msg);
        } else {
            List<ValuesClause> valueClauseList = insertStmt.getValuesList();
            Map<Integer, List<ValuesClause>> nodeValuesMap = new HashMap<Integer, List<ValuesClause>>();
            Map<Integer, Integer> slotsMap = new HashMap<>();
            TableConfig tableConfig = schema.getTables().get(tableName);
            AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
            for (ValuesClause valueClause : valueClauseList) {
                if (valueClause.getValues().size() != columnNum) {
                    String msg = "bad insert sql columnSize != valueSize:" + columnNum + " != " + valueClause.getValues().size() + "values:" + valueClause;
                    LOGGER.warn(msg);
                    throw new SQLNonTransientException(msg);
                }
                SQLExpr expr = valueClause.getValues().get(shardingColIndex);
                String shardingValue = null;
                if (expr instanceof SQLIntegerExpr) {
                    SQLIntegerExpr intExpr = (SQLIntegerExpr) expr;
                    shardingValue = intExpr.getNumber() + "";
                } else if (expr instanceof SQLCharExpr) {
                    SQLCharExpr charExpr = (SQLCharExpr) expr;
                    shardingValue = charExpr.getText();
                }
                Integer nodeIndex = algorithm.calculate(shardingValue);
                if (algorithm instanceof SlotFunction) {
                    slotsMap.put(nodeIndex, ((SlotFunction) algorithm).slotValue());
                }
                //没找到插入的分片
                if (nodeIndex == null) {
                    String msg = "can't find any valid datanode :" + tableName + " -> " + partitionColumn + " -> " + shardingValue;
                    LOGGER.warn(msg);
                    throw new SQLNonTransientException(msg);
                }
                if (nodeValuesMap.get(nodeIndex) == null) {
                    nodeValuesMap.put(nodeIndex, new ArrayList<ValuesClause>());
                }
                nodeValuesMap.get(nodeIndex).add(valueClause);
            }
            RouteResultsetNode[] nodes = new RouteResultsetNode[nodeValuesMap.size()];
            int count = 0;
            for (Map.Entry<Integer, List<ValuesClause>> node : nodeValuesMap.entrySet()) {
                Integer nodeIndex = node.getKey();
                List<ValuesClause> valuesList = node.getValue();
                insertStmt.setValuesList(valuesList);
                nodes[count] = new RouteResultsetNode(tableConfig.getDataNodes().get(nodeIndex), rrs.getSqlType(), insertStmt.toString());
                if (algorithm instanceof SlotFunction) {
                    nodes[count].setSlot(slotsMap.get(nodeIndex));
                    nodes[count].setStatement(ParseUtil.changeInsertAddSlot(nodes[count].getStatement(), nodes[count].getSlot()));
                }
                nodes[count++].setSource(rrs);
            }
            rrs.setNodes(nodes);
            rrs.setFinishedRoute(true);
        }
    } else if (insertStmt.getQuery() != null) {
        // insert into .... select ....
        String msg = "TODO:insert into .... select .... not supported!";
        LOGGER.warn(msg);
        throw new SQLNonTransientException(msg);
    }
}

Example 9 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class RouterUtil method routeToDistTableNode.

private static RouteResultset routeToDistTableNode(String tableName, SchemaConfig schema, RouteResultset rrs, String orgSql, Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions, LayerCachePool cachePool, boolean isSelect) throws SQLNonTransientException {
    TableConfig tableConfig = schema.getTables().get(tableName);
    if (tableConfig == null) {
        String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
        LOGGER.warn(msg);
        throw new SQLNonTransientException(msg);
    }
    if (tableConfig.isGlobalTable()) {
        String msg = "can't suport district table  " + tableName + " schema:" + schema.getName() + " for global table ";
        LOGGER.warn(msg);
        throw new SQLNonTransientException(msg);
    }
    String partionCol = tableConfig.getPartitionColumn();
    //		String primaryKey = tableConfig.getPrimaryKey();
    boolean isLoadData = false;
    Set<String> tablesRouteSet = new HashSet<String>();
    List<String> dataNodes = tableConfig.getDataNodes();
    if (dataNodes.size() > 1) {
        String msg = "can't suport district table  " + tableName + " schema:" + schema.getName() + " for mutiple dataNode " + dataNodes;
        LOGGER.warn(msg);
        throw new SQLNonTransientException(msg);
    }
    String dataNode = dataNodes.get(0);
    //主键查找缓存暂时不实现
    if (tablesAndConditions.isEmpty()) {
        List<String> subTables = tableConfig.getDistTables();
        tablesRouteSet.addAll(subTables);
    }
    for (Map.Entry<String, Map<String, Set<ColumnRoutePair>>> entry : tablesAndConditions.entrySet()) {
        boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null;
        Map<String, Set<ColumnRoutePair>> columnsMap = entry.getValue();
        Set<ColumnRoutePair> partitionValue = columnsMap.get(partionCol);
        if (partitionValue == null || partitionValue.size() == 0) {
            tablesRouteSet.addAll(tableConfig.getDistTables());
        } else {
            for (ColumnRoutePair pair : partitionValue) {
                AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
                if (pair.colValue != null) {
                    Integer tableIndex = algorithm.calculate(pair.colValue);
                    if (tableIndex == null) {
                        String msg = "can't find any valid datanode :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
                        LOGGER.warn(msg);
                        throw new SQLNonTransientException(msg);
                    }
                    String subTable = tableConfig.getDistTables().get(tableIndex);
                    if (subTable != null) {
                        tablesRouteSet.add(subTable);
                        if (algorithm instanceof SlotFunction) {
                            rrs.getDataNodeSlotMap().put(subTable, ((SlotFunction) algorithm).slotValue());
                        }
                    }
                }
                if (pair.rangeValue != null) {
                    Integer[] tableIndexs = algorithm.calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
                    for (Integer idx : tableIndexs) {
                        String subTable = tableConfig.getDistTables().get(idx);
                        if (subTable != null) {
                            tablesRouteSet.add(subTable);
                            if (algorithm instanceof SlotFunction) {
                                rrs.getDataNodeSlotMap().put(subTable, ((SlotFunction) algorithm).slotValue());
                            }
                        }
                    }
                }
            }
        }
    }
    Object[] subTables = tablesRouteSet.toArray();
    RouteResultsetNode[] nodes = new RouteResultsetNode[subTables.length];
    Map<String, Integer> dataNodeSlotMap = rrs.getDataNodeSlotMap();
    for (int i = 0; i < nodes.length; i++) {
        String table = String.valueOf(subTables[i]);
        String changeSql = orgSql;
        //rrs.getStatement()
        nodes[i] = new RouteResultsetNode(dataNode, rrs.getSqlType(), changeSql);
        nodes[i].setSubTableName(table);
        nodes[i].setSource(rrs);
        if (rrs.getDataNodeSlotMap().containsKey(dataNode)) {
            nodes[i].setSlot(rrs.getDataNodeSlotMap().get(dataNode));
        }
        if (rrs.getCanRunInReadDB() != null) {
            nodes[i].setCanRunInReadDB(rrs.getCanRunInReadDB());
        }
        if (dataNodeSlotMap.containsKey(table)) {
            nodes[i].setSlot(dataNodeSlotMap.get(table));
        }
        if (rrs.getRunOnSlave() != null) {
            nodes[0].setRunOnSlave(rrs.getRunOnSlave());
        }
    }
    rrs.setNodes(nodes);
    rrs.setSubTables(tablesRouteSet);
    rrs.setFinishedRoute(true);
    return rrs;
}

Example 10 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class RouterUtil method findRouteWithcConditionsForTables.

/**
	 * 处理分库表路由
	 */
public static void findRouteWithcConditionsForTables(SchemaConfig schema, RouteResultset rrs, Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions, Map<String, Set<String>> tablesRouteMap, String sql, LayerCachePool cachePool, boolean isSelect) throws SQLNonTransientException {
    //为分库表找路由
    for (Map.Entry<String, Map<String, Set<ColumnRoutePair>>> entry : tablesAndConditions.entrySet()) {
        String tableName = entry.getKey().toUpperCase();
        TableConfig tableConfig = schema.getTables().get(tableName);
        if (tableConfig == null) {
            String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
            LOGGER.warn(msg);
            throw new SQLNonTransientException(msg);
        }
        if (tableConfig.getDistTables() != null && tableConfig.getDistTables().size() > 0) {
            routeToDistTableNode(tableName, schema, rrs, sql, tablesAndConditions, cachePool, isSelect);
        }
        //全局表或者不分库的表略过（全局表后面再计算）
        if (tableConfig.isGlobalTable() || schema.getTables().get(tableName).getDataNodes().size() == 1) {
            continue;
        } else {
            //非全局表：分库表、childTable、其他
            Map<String, Set<ColumnRoutePair>> columnsMap = entry.getValue();
            String joinKey = tableConfig.getJoinKey();
            String partionCol = tableConfig.getPartitionColumn();
            String primaryKey = tableConfig.getPrimaryKey();
            boolean isFoundPartitionValue = partionCol != null && entry.getValue().get(partionCol) != null;
            boolean isLoadData = false;
            if (LOGGER.isDebugEnabled() && sql.startsWith(LoadData.loadDataHint) || rrs.isLoadData()) {
                //由于load data一次会计算很多路由数据，如果输出此日志会极大降低load data的性能
                isLoadData = true;
            }
            if (entry.getValue().get(primaryKey) != null && entry.getValue().size() == 1 && !isLoadData) {
                //主键查找
                // try by primary key if found in cache
                Set<ColumnRoutePair> primaryKeyPairs = entry.getValue().get(primaryKey);
                if (primaryKeyPairs != null) {
                    if (LOGGER.isDebugEnabled()) {
                        LOGGER.debug("try to find cache by primary key ");
                    }
                    String tableKey = schema.getName() + '_' + tableName;
                    boolean allFound = true;
                    for (ColumnRoutePair pair : primaryKeyPairs) {
                        //可能id in(1,2,3)多主键
                        String cacheKey = pair.colValue;
                        String dataNode = (String) cachePool.get(tableKey, cacheKey);
                        if (dataNode == null) {
                            allFound = false;
                            continue;
                        } else {
                            if (tablesRouteMap.get(tableName) == null) {
                                tablesRouteMap.put(tableName, new HashSet<String>());
                            }
                            tablesRouteMap.get(tableName).add(dataNode);
                            continue;
                        }
                    }
                    if (!allFound) {
                        // need cache primary key ->datanode relation
                        if (isSelect && tableConfig.getPrimaryKey() != null) {
                            rrs.setPrimaryKey(tableKey + '.' + tableConfig.getPrimaryKey());
                        }
                    } else {
                        //主键缓存中找到了就执行循环的下一轮
                        continue;
                    }
                }
            }
            if (isFoundPartitionValue) {
                //分库表
                Set<ColumnRoutePair> partitionValue = columnsMap.get(partionCol);
                if (partitionValue == null || partitionValue.size() == 0) {
                    if (tablesRouteMap.get(tableName) == null) {
                        tablesRouteMap.put(tableName, new HashSet<String>());
                    }
                    tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
                } else {
                    for (ColumnRoutePair pair : partitionValue) {
                        AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
                        if (pair.colValue != null) {
                            Integer nodeIndex = algorithm.calculate(pair.colValue);
                            if (nodeIndex == null) {
                                String msg = "can't find any valid datanode :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue;
                                LOGGER.warn(msg);
                                throw new SQLNonTransientException(msg);
                            }
                            ArrayList<String> dataNodes = tableConfig.getDataNodes();
                            String node;
                            if (nodeIndex >= 0 && nodeIndex < dataNodes.size()) {
                                node = dataNodes.get(nodeIndex);
                            } else {
                                node = null;
                                String msg = "Can't find a valid data node for specified node index :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn() + " -> " + pair.colValue + " -> " + "Index : " + nodeIndex;
                                LOGGER.warn(msg);
                                throw new SQLNonTransientException(msg);
                            }
                            if (node != null) {
                                if (tablesRouteMap.get(tableName) == null) {
                                    tablesRouteMap.put(tableName, new HashSet<String>());
                                }
                                if (algorithm instanceof SlotFunction) {
                                    rrs.getDataNodeSlotMap().put(node, ((SlotFunction) algorithm).slotValue());
                                }
                                tablesRouteMap.get(tableName).add(node);
                            }
                        }
                        if (pair.rangeValue != null) {
                            Integer[] nodeIndexs = algorithm.calculateRange(pair.rangeValue.beginValue.toString(), pair.rangeValue.endValue.toString());
                            ArrayList<String> dataNodes = tableConfig.getDataNodes();
                            String node;
                            for (Integer idx : nodeIndexs) {
                                if (idx >= 0 && idx < dataNodes.size()) {
                                    node = dataNodes.get(idx);
                                } else {
                                    String msg = "Can't find valid data node(s) for some of specified node indexes :" + tableConfig.getName() + " -> " + tableConfig.getPartitionColumn();
                                    LOGGER.warn(msg);
                                    throw new SQLNonTransientException(msg);
                                }
                                if (node != null) {
                                    if (tablesRouteMap.get(tableName) == null) {
                                        tablesRouteMap.put(tableName, new HashSet<String>());
                                    }
                                    if (algorithm instanceof SlotFunction) {
                                        rrs.getDataNodeSlotMap().put(node, ((SlotFunction) algorithm).slotValue());
                                    }
                                    tablesRouteMap.get(tableName).add(node);
                                }
                            }
                        }
                    }
                }
            } else if (joinKey != null && columnsMap.get(joinKey) != null && columnsMap.get(joinKey).size() != 0) {
                //childTable  (如果是select 语句的父子表join)之前要找到root table,将childTable移除,只留下root table
                Set<ColumnRoutePair> joinKeyValue = columnsMap.get(joinKey);
                Set<String> dataNodeSet = ruleByJoinValueCalculate(rrs, tableConfig, joinKeyValue);
                if (dataNodeSet.isEmpty()) {
                    throw new SQLNonTransientException("parent key can't find any valid datanode ");
                }
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("found partion nodes (using parent partion rule directly) for child table to update  " + Arrays.toString(dataNodeSet.toArray()) + " sql :" + sql);
                }
                if (dataNodeSet.size() > 1) {
                    routeToMultiNode(rrs.isCacheAble(), rrs, dataNodeSet, sql);
                    rrs.setFinishedRoute(true);
                    return;
                } else {
                    rrs.setCacheAble(true);
                    routeToSingleNode(rrs, dataNodeSet.iterator().next(), sql);
                    return;
                }
            } else {
                //没找到拆分字段，该表的所有节点都路由
                if (tablesRouteMap.get(tableName) == null) {
                    tablesRouteMap.put(tableName, new HashSet<String>());
                }
                boolean isSlotFunction = tableConfig.getRule() != null && tableConfig.getRule().getRuleAlgorithm() instanceof SlotFunction;
                if (isSlotFunction) {
                    for (String dn : tableConfig.getDataNodes()) {
                        rrs.getDataNodeSlotMap().put(dn, -1);
                    }
                }
                tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
            }
        }
    }
}