Examples with TableConfig com.actiontech.dble.config.model.TableConfig used on opensource projects

Example 16 with TableConfig

use of com.actiontech.dble.config.model.TableConfig in project dble by actiontech.

the class ShowTables method getTableSet.

public static Map<String, String> getTableSet(String cSchema, ShowCreateStmtInfo info) {
    // remove the table which is not created but configured
    SchemaMeta schemata = DbleServer.getInstance().getTmManager().getCatalogs().get(cSchema);
    if (schemata == null) {
        return new HashMap<>();
    }
    Map meta = schemata.getTableMetas();
    TreeMap<String, String> tableMap = new TreeMap<>();
    Map<String, SchemaConfig> schemas = DbleServer.getInstance().getConfig().getSchemas();
    if (null != info.getLike()) {
        String p = "^" + info.getLike().replaceAll("%", ".*");
        Pattern pattern = Pattern.compile(p, Pattern.CASE_INSENSITIVE);
        Matcher maLike;
        for (TableConfig tbConfig : schemas.get(cSchema).getTables().values()) {
            String tbName = tbConfig.getName();
            maLike = pattern.matcher(tbName);
            if (maLike.matches() && meta.get(tbName) != null) {
                String tbType = tbConfig.getTableType() == TableConfig.TableTypeEnum.TYPE_GLOBAL_TABLE ? "GLOBAL TABLE" : "SHARDING TABLE";
                tableMap.put(tbName, tbType);
            }
        }
    } else {
        for (TableConfig tbConfig : schemas.get(cSchema).getTables().values()) {
            String tbName = tbConfig.getName();
            if (meta.get(tbName) != null) {
                String tbType = tbConfig.getTableType() == TableConfig.TableTypeEnum.TYPE_GLOBAL_TABLE ? "GLOBAL TABLE" : "SHARDING TABLE";
                tableMap.put(tbName, tbType);
            }
        }
    }
    return tableMap;
}

Example 17 with TableConfig

use of com.actiontech.dble.config.model.TableConfig in project dble by actiontech.

the class GlobalTableUtil method consistencyCheck.

public static void consistencyCheck() {
    ServerConfig config = DbleServer.getInstance().getConfig();
    for (TableConfig table : globalTableMap.values()) {
        Map<String, ArrayList<PhysicalDBNode>> executedMap = new HashMap<>();
        // <table name="travelrecord" dataNode="dn1,dn2,dn3"
        for (String nodeName : table.getDataNodes()) {
            Map<String, PhysicalDBNode> map = config.getDataNodes();
            for (PhysicalDBNode dbNode : map.values()) {
                // <dataNode name="dn1" dataHost="localhost1" database="db1" />
                if (nodeName.equals(dbNode.getName())) {
                    // dn1,dn2,dn3
                    PhysicalDBPool pool = dbNode.getDbPool();
                    Collection<PhysicalDatasource> allDS = pool.getAllDataSources();
                    for (PhysicalDatasource pds : allDS) {
                        if (pds instanceof MySQLDataSource) {
                            ArrayList<PhysicalDBNode> nodes = executedMap.get(pds.getName());
                            if (nodes == null) {
                                nodes = new ArrayList<>();
                            }
                            nodes.add(dbNode);
                            executedMap.put(pds.getName(), nodes);
                        }
                    }
                }
            }
        }
        for (Map.Entry<String, ArrayList<PhysicalDBNode>> entry : executedMap.entrySet()) {
            ArrayList<PhysicalDBNode> nodes = entry.getValue();
            String[] schemas = new String[nodes.size()];
            for (int index = 0; index < nodes.size(); index++) {
                schemas[index] = StringUtil.removeBackQuote(nodes.get(index).getDatabase());
            }
            Collection<PhysicalDatasource> allDS = nodes.get(0).getDbPool().getAllDataSources();
            for (PhysicalDatasource pds : allDS) {
                if (pds instanceof MySQLDataSource && entry.getKey().equals(pds.getName())) {
                    MySQLDataSource mds = (MySQLDataSource) pds;
                    MySQLConsistencyChecker checker = new MySQLConsistencyChecker(mds, schemas, table.getName());
                    isInnerColumnCheckFinished = 0;
                    checker.checkInnerColumnExist();
                    while (isInnerColumnCheckFinished <= 0) {
                        LOGGER.debug("isInnerColumnCheckFinished:" + isInnerColumnCheckFinished);
                        try {
                            TimeUnit.SECONDS.sleep(1);
                        } catch (InterruptedException e) {
                            LOGGER.info(e.getMessage());
                        }
                    }
                    LOGGER.debug("isInnerColumnCheckFinished:" + isInnerColumnCheckFinished);
                    // check another measure
                    checker = new MySQLConsistencyChecker(mds, schemas, table.getName());
                    isColumnCountCheckFinished = 0;
                    checker.checkRecordCount();
                    while (isColumnCountCheckFinished <= 0) {
                        LOGGER.debug("isColumnCountCheckFinished:" + isColumnCountCheckFinished);
                        try {
                            TimeUnit.SECONDS.sleep(1);
                        } catch (InterruptedException e) {
                            LOGGER.info(e.getMessage());
                        }
                    }
                    LOGGER.debug("isColumnCountCheckFinished:" + isColumnCountCheckFinished);
                    checker = new MySQLConsistencyChecker(mds, schemas, table.getName());
                    checker.checkMaxTimeStamp();
                }
            }
        }
    }
}

Example 18 with TableConfig

use of com.actiontech.dble.config.model.TableConfig in project dble by actiontech.

the class RouterUtil method findRouterWithConditionsForTables.

/**
 * findRouterWithConditionsForTables
 */
public static void findRouterWithConditionsForTables(SchemaConfig schema, RouteResultset rrs, Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions, Map<String, Set<String>> tablesRouteMap, LayerCachePool cachePool, boolean isSelect, boolean isSingleTable) throws SQLNonTransientException {
    // router for shard-ing tables
    for (Map.Entry<String, Map<String, Set<ColumnRoutePair>>> entry : tablesAndConditions.entrySet()) {
        String tableName = entry.getKey();
        if (DbleServer.getInstance().getSystemVariables().isLowerCaseTableNames()) {
            tableName = tableName.toLowerCase();
        }
        if (tableName.startsWith(schema.getName() + ".")) {
            tableName = tableName.substring(schema.getName().length() + 1);
        }
        TableConfig tableConfig = schema.getTables().get(tableName);
        if (tableConfig == null) {
            if (isSingleTable) {
                String msg = "can't find table [" + tableName + "[ define in schema " + ":" + schema.getName();
                LOGGER.info(msg);
                throw new SQLNonTransientException(msg);
            } else {
                // cross to other schema
                continue;
            }
        }
        if (tableConfig.isGlobalTable() || schema.getTables().get(tableName).getDataNodes().size() == 1) {
            // global table or single node shard-ing table will router later
            continue;
        } else {
            // shard-ing table,childTable or others
            Map<String, Set<ColumnRoutePair>> columnsMap = entry.getValue();
            if (tryRouteWithPrimaryCache(rrs, tablesRouteMap, cachePool, columnsMap, schema, tableName, tableConfig.getPrimaryKey(), isSelect)) {
                continue;
            }
            String joinKey = tableConfig.getJoinKey();
            String partitionCol = tableConfig.getPartitionColumn();
            boolean isFoundPartitionValue = partitionCol != null && columnsMap.get(partitionCol) != null;
            // where filter contains partition column
            if (isFoundPartitionValue) {
                Set<ColumnRoutePair> partitionValue = columnsMap.get(partitionCol);
                if (partitionValue.size() == 0) {
                    if (tablesRouteMap.get(tableName) == null) {
                        tablesRouteMap.put(tableName, new HashSet<String>());
                    }
                    tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
                } else {
                    routeWithPartition(tablesRouteMap, tableName, tableConfig, partitionValue);
                }
            } else if (joinKey != null && columnsMap.get(joinKey) != null && columnsMap.get(joinKey).size() != 0) {
                routerForJoinTable(rrs, tableConfig, columnsMap, joinKey);
                return;
            } else {
                // no partition column,router to all nodes
                if (tablesRouteMap.get(tableName) == null) {
                    tablesRouteMap.put(tableName, new HashSet<String>());
                }
                tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
            }
        }
    }
}

Example 19 with TableConfig

use of com.actiontech.dble.config.model.TableConfig in project dble by actiontech.

the class RouterUtil method tryRouteForTables.

/**
 * tryRouteFor multiTables
 */
public static RouteResultset tryRouteForTables(SchemaConfig schema, DruidShardingParseInfo ctx, RouteCalculateUnit routeUnit, RouteResultset rrs, boolean isSelect, LayerCachePool cachePool) throws SQLException {
    List<String> tables = ctx.getTables();
    // no sharding table
    if (isNoSharding(schema, tables.get(0))) {
        return routeToSingleNode(rrs, schema.getDataNode());
    }
    if (tables.size() == 1) {
        return RouterUtil.tryRouteForOneTable(schema, routeUnit, tables.get(0), rrs, isSelect, cachePool);
    }
    /**
     * multi-table it must be ER OR   global* normal , global* er
     */
    // map <table,data_nodes>
    Map<String, Set<String>> tablesRouteMap = new HashMap<>();
    Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions = routeUnit.getTablesAndConditions();
    if (tablesAndConditions != null && tablesAndConditions.size() > 0) {
        // findRouter for shard-ing table
        RouterUtil.findRouterWithConditionsForTables(schema, rrs, tablesAndConditions, tablesRouteMap, cachePool, isSelect, false);
        if (rrs.isFinishedRoute()) {
            return rrs;
        }
    }
    // if global table and normal table has no intersection ,they had treat as normal join
    for (String tableName : tables) {
        TableConfig tableConfig = schema.getTables().get(tableName);
        if (tableConfig != null && !tableConfig.isGlobalTable() && tablesRouteMap.get(tableName) == null) {
            // the other is single table
            tablesRouteMap.put(tableName, new HashSet<String>());
            tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
        }
    }
    Set<String> retNodesSet = new HashSet<>();
    boolean isFirstAdd = true;
    for (Map.Entry<String, Set<String>> entry : tablesRouteMap.entrySet()) {
        if (entry.getValue() == null || entry.getValue().size() == 0) {
            throw new SQLNonTransientException("parent key can't find any valid datanode ");
        } else {
            if (isFirstAdd) {
                retNodesSet.addAll(entry.getValue());
                isFirstAdd = false;
            } else {
                retNodesSet.retainAll(entry.getValue());
                if (retNodesSet.size() == 0) {
                    // two tables has no no intersection
                    String errMsg = "invalid route in sql, multi tables found but datanode has no intersection " + " sql:" + rrs.getStatement();
                    LOGGER.info(errMsg);
                    throw new SQLNonTransientException(errMsg);
                }
            }
        }
    }
    // retNodesSet.size() >0
    routeToMultiNode(isSelect, rrs, retNodesSet);
    return rrs;
}

Example 20 with TableConfig

use of com.actiontech.dble.config.model.TableConfig in project dble by actiontech.

the class RouterUtil method tryRouteForOneTable.

/**
 * tryRouteForOneTable
 */
public static RouteResultset tryRouteForOneTable(SchemaConfig schema, RouteCalculateUnit routeUnit, String tableName, RouteResultset rrs, boolean isSelect, LayerCachePool cachePool) throws SQLException {
    TableConfig tc = schema.getTables().get(tableName);
    if (tc == null) {
        String msg = "Table '" + schema.getName() + "." + tableName + "' doesn't exist";
        throw new SQLException(msg, "42S02", ErrorCode.ER_NO_SUCH_TABLE);
    }
    if (tc.isGlobalTable()) {
        if (isSelect) {
            // global select ,not cache route result
            rrs.setCacheAble(false);
            rrs.setGlobalTable(true);
            return routeToSingleNode(rrs, tc.getRandomDataNode());
        } else {
            // insert into all global table's node
            return routeToMultiNode(false, rrs, tc.getDataNodes(), true);
        }
    } else {
        // single table or shard-ing table
        if (!checkRuleRequired(schema, routeUnit, tc)) {
            throw new IllegalArgumentException("route rule for table " + tc.getName() + " is required: " + rrs.getStatement());
        }
        if ((tc.getPartitionColumn() == null && tc.getParentTC() == null) || (tc.getParentTC() != null && tc.getDirectRouteTC() == null)) {
            // single table or one of the children of complex ER table
            return routeToMultiNode(rrs.isCacheAble(), rrs, tc.getDataNodes());
        } else {
            Map<String, Set<String>> tablesRouteMap = new HashMap<>();
            if (routeUnit.getTablesAndConditions() != null && routeUnit.getTablesAndConditions().size() > 0) {
                RouterUtil.findRouterWithConditionsForTables(schema, rrs, routeUnit.getTablesAndConditions(), tablesRouteMap, cachePool, isSelect, true);
                if (rrs.isFinishedRoute()) {
                    return rrs;
                }
            }
            if (tablesRouteMap.get(tableName) == null) {
                return routeToMultiNode(rrs.isCacheAble(), rrs, tc.getDataNodes());
            } else {
                return routeToMultiNode(rrs.isCacheAble(), rrs, tablesRouteMap.get(tableName));
            }
        }
    }
}