Examples with AbstractPartitionAlgorithm io.mycat.route.function.AbstractPartitionAlgorithm used on opensource projects

Example 1 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class MigrateHandler method handle.

public static void handle(String stmt, ServerConnection c) {
    Map<String, String> map = parse(stmt);
    String table = map.get("table");
    String add = map.get("add");
    if (table == null) {
        writeErrMessage(c, "table cannot be null");
        return;
    }
    if (add == null) {
        writeErrMessage(c, "add cannot be null");
        return;
    }
    String taskID = getUUID();
    try {
        SchemaConfig schemaConfig = MycatServer.getInstance().getConfig().getSchemas().get(c.getSchema());
        TableConfig tableConfig = schemaConfig.getTables().get(table.toUpperCase());
        AbstractPartitionAlgorithm algorithm = tableConfig.getRule().getRuleAlgorithm();
        if (!(algorithm instanceof PartitionByCRC32PreSlot)) {
            writeErrMessage(c, "table: " + table + " rule is not be PartitionByCRC32PreSlot");
            return;
        }
        Map<Integer, List<Range>> integerListMap = ((PartitionByCRC32PreSlot) algorithm).getRangeMap();
        integerListMap = (Map<Integer, List<Range>>) ObjectUtil.copyObject(integerListMap);
        ArrayList<String> oldDataNodes = tableConfig.getDataNodes();
        List<String> newDataNodes = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(add);
        Map<String, List<MigrateTask>> tasks = MigrateUtils.balanceExpand(table, integerListMap, oldDataNodes, newDataNodes, PartitionByCRC32PreSlot.DEFAULT_SLOTS_NUM);
        CuratorTransactionFinal transactionFinal = null;
        String taskBase = ZKUtils.getZKBasePath() + "migrate/" + c.getSchema();
        String taskPath = taskBase + "/" + taskID;
        CuratorFramework client = ZKUtils.getConnection();
        //校验 之前同一个表的迁移任务未完成，则jzhi禁止继续
        if (client.checkExists().forPath(taskBase) != null) {
            List<String> childTaskList = client.getChildren().forPath(taskBase);
            for (String child : childTaskList) {
                TaskNode taskNode = JSON.parseObject(ZKUtils.getConnection().getData().forPath(taskBase + "/" + child), TaskNode.class);
                if (taskNode.getSchema().equalsIgnoreCase(c.getSchema()) && table.equalsIgnoreCase(taskNode.getTable()) && taskNode.getStatus() < 5) {
                    writeErrMessage(c, "table: " + table + " previous migrate task is still running,on the same time one table only one task");
                    return;
                }
            }
        }
        client.create().creatingParentsIfNeeded().forPath(taskPath);
        TaskNode taskNode = new TaskNode();
        taskNode.setSchema(c.getSchema());
        taskNode.setSql(stmt);
        taskNode.setTable(table);
        taskNode.setAdd(add);
        taskNode.setStatus(0);
        Map<String, Integer> fromNodeSlaveIdMap = new HashMap<>();
        List<MigrateTask> allTaskList = new ArrayList<>();
        for (Map.Entry<String, List<MigrateTask>> entry : tasks.entrySet()) {
            String key = entry.getKey();
            List<MigrateTask> value = entry.getValue();
            for (MigrateTask migrateTask : value) {
                migrateTask.setSchema(c.getSchema());
                //分配slaveid只需要一个dataHost分配一个即可，后续任务执行模拟从节点只需要一个dataHost一个
                String dataHost = getDataHostNameFromNode(migrateTask.getFrom());
                if (fromNodeSlaveIdMap.containsKey(dataHost)) {
                    migrateTask.setSlaveId(fromNodeSlaveIdMap.get(dataHost));
                } else {
                    migrateTask.setSlaveId(getSlaveIdFromZKForDataNode(migrateTask.getFrom()));
                    fromNodeSlaveIdMap.put(dataHost, migrateTask.getSlaveId());
                }
            }
            allTaskList.addAll(value);
        }
        transactionFinal = client.inTransaction().setData().forPath(taskPath, JSON.toJSONBytes(taskNode)).and();
        //合并成dataHost级别任务
        Map<String, List<MigrateTask>> dataHostMigrateMap = mergerTaskForDataHost(allTaskList);
        for (Map.Entry<String, List<MigrateTask>> entry : dataHostMigrateMap.entrySet()) {
            String key = entry.getKey();
            List<MigrateTask> value = entry.getValue();
            String path = taskPath + "/" + key;
            transactionFinal = transactionFinal.create().forPath(path, JSON.toJSONBytes(value)).and();
        }
        transactionFinal.commit();
    } catch (Exception e) {
        LOGGER.error("migrate error", e);
        writeErrMessage(c, "migrate error:" + e);
        return;
    }
    writePackToClient(c, taskID);
    LOGGER.info("task start", new Date());
}

Example 2 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class ConfigComparer method checkRuleConfig.

//校验前后路由规则是否一致
private void checkRuleConfig(RuleConfig oldRC, RuleConfig newRC, String schemaName, String tableName) {
    if (!oldRC.getColumn().equalsIgnoreCase(newRC.getColumn())) {
        throw new ConfigException(schemaName + ":" + tableName + " old & new partition column is not same!");
    }
    AbstractPartitionAlgorithm oldAlg = oldRC.getRuleAlgorithm();
    AbstractPartitionAlgorithm newAlg = newRC.getRuleAlgorithm();
    //判断路由算法前后是否一致
    if (!oldAlg.getClass().isAssignableFrom(newAlg.getClass())) {
        throw new ConfigException(schemaName + ":" + tableName + " old & new rule Algorithm is not same!");
    }
}

Example 3 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class XMLRuleLoader method loadTableRules.

/**
	 * tableRule标签结构：
	 * <tableRule name="sharding-by-month">
	 *     <rule>
	 *         <columns>create_date</columns>
	 *         <algorithm>partbymonth</algorithm>
	 *     </rule>
	 * </tableRule>
	 * @param root
	 * @throws SQLSyntaxErrorException
     */
private void loadTableRules(Element root) throws SQLSyntaxErrorException {
    //获取每个tableRule标签
    NodeList list = root.getElementsByTagName("tableRule");
    for (int i = 0, n = list.getLength(); i < n; ++i) {
        Node node = list.item(i);
        if (node instanceof Element) {
            Element e = (Element) node;
            //先判断是否重复
            String name = e.getAttribute("name");
            if (tableRules.containsKey(name)) {
                throw new ConfigException("table rule " + name + " duplicated!");
            }
            //获取rule标签
            NodeList ruleNodes = e.getElementsByTagName("rule");
            int length = ruleNodes.getLength();
            if (length > 1) {
                throw new ConfigException("only one rule can defined :" + name);
            }
            //目前只处理第一个，未来可能有多列复合逻辑需求
            //RuleConfig是保存着rule与function对应关系的对象
            RuleConfig rule = loadRule((Element) ruleNodes.item(0));
            String funName = rule.getFunctionName();
            //判断function是否存在，获取function
            AbstractPartitionAlgorithm func = functions.get(funName);
            if (func == null) {
                throw new ConfigException("can't find function of name :" + funName);
            }
            rule.setRuleAlgorithm(func);
            //保存到tableRules
            tableRules.put(name, new TableRuleConfig(name, rule));
        }
    }
}

Example 4 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class XMLRuleLoader method loadFunctions.

/**
	 * function标签结构：
	 * <function name="partbymonth" class="io.mycat.route.function.PartitionByMonth">
	 *     <property name="dateFormat">yyyy-MM-dd</property>
	 *     <property name="sBeginDate">2015-01-01</property>
	 * </function>
	 * @param root
	 * @throws ClassNotFoundException
	 * @throws InstantiationException
	 * @throws IllegalAccessException
	 * @throws InvocationTargetException
     */
private void loadFunctions(Element root) throws ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException {
    NodeList list = root.getElementsByTagName("function");
    for (int i = 0, n = list.getLength(); i < n; ++i) {
        Node node = list.item(i);
        if (node instanceof Element) {
            Element e = (Element) node;
            //获取name标签
            String name = e.getAttribute("name");
            //如果Map已有，则function重复
            if (functions.containsKey(name)) {
                throw new ConfigException("rule function " + name + " duplicated!");
            }
            //获取class标签
            String clazz = e.getAttribute("class");
            //根据class利用反射新建分片算法
            AbstractPartitionAlgorithm function = createFunction(name, clazz);
            //根据读取参数配置分片算法
            ParameterMapping.mapping(function, ConfigUtil.loadElements(e));
            //每个AbstractPartitionAlgorithm可能会实现init来初始化
            function.init();
            //放入functions map
            functions.put(name, function);
        }
    }
}

Example 5 with AbstractPartitionAlgorithm

use of io.mycat.route.function.AbstractPartitionAlgorithm in project Mycat-Server by MyCATApache.

the class XMLSchemaLoader method checkRuleSuitTable.

/**
	 * 检查分片表分片规则配置, 目前主要检查分片表分片算法定义与分片dataNode是否匹配<br>
	 * 例如分片表定义如下:<br>
	 * {@code
	 * <table name="hotnews" primaryKey="ID" autoIncrement="true" dataNode="dn1,dn2"
			   rule="mod-long" />
	 * }
	 * <br>
	 * 分片算法如下:<br>
	 * {@code
	 * <function name="mod-long" class="io.mycat.route.function.PartitionByMod">
		<!-- how many data nodes -->
		<property name="count">3</property>
	   </function>
	 * }
	 * <br>
	 * shard table datanode(2) < function count(3) 此时检测为不匹配
	 */
private void checkRuleSuitTable(TableConfig tableConf) {
    AbstractPartitionAlgorithm function = tableConf.getRule().getRuleAlgorithm();
    int suitValue = function.suitableFor(tableConf);
    switch(suitValue) {
        case -1:
            // 少节点,给提示并抛异常
            throw new ConfigException("Illegal table conf : table [ " + tableConf.getName() + " ] rule function [ " + tableConf.getRule().getFunctionName() + " ] partition size : " + tableConf.getRule().getRuleAlgorithm().getPartitionNum() + " > table datanode size : " + tableConf.getDataNodes().size() + ", please make sure table datanode size = function partition size");
        case 0:
            // table datanode size == rule function partition size
            break;
        case 1:
            // 有些节点是多余的,给出warn log
            LOGGER.warn("table conf : table [ {} ] rule function [ {} ] partition size : {} < table datanode size : {} , this cause some datanode to be redundant", new String[] { tableConf.getName(), tableConf.getRule().getFunctionName(), String.valueOf(tableConf.getRule().getRuleAlgorithm().getPartitionNum()), String.valueOf(tableConf.getDataNodes().size()) });
            break;
    }
}