Examples with ShardingTable io.mycat.calcite.table.ShardingTable used on opensource projects

Example 1 with ShardingTable

use of io.mycat.calcite.table.ShardingTable in project Mycat2 by MyCATApache.

the class Distribution method join.

public Optional<Distribution> join(Distribution arg) {
    switch(arg.type()) {
        case PHY:
            switch(this.type()) {
                case PHY:
                    NormalTable leftNormalTable = this.normalTables.get(0);
                    NormalTable rightNormalTable = arg.normalTables.get(0);
                    if (leftNormalTable.getDataNode().getTargetName().equals(rightNormalTable.getDataNode().getTargetName())) {
                        return Optional.of(new Distribution(this.shardingTables, this.globalTables, merge(this.normalTables, arg.normalTables)));
                    }
                    return Optional.empty();
                case BROADCAST:
                    return Optional.of(new Distribution(merge(this.shardingTables, arg.shardingTables), merge(this.globalTables, arg.globalTables), merge(this.normalTables, arg.normalTables)));
                case SHARDING:
                    ShardingTable shardingTable = this.shardingTables.get(0);
                    if (shardingTable.function().isAllPartitionInTargetName(arg.normalTables.get(0).getDataNode().getTargetName())) {
                        return Optional.of(new Distribution(merge(this.shardingTables, arg.shardingTables), merge(this.globalTables, arg.globalTables), merge(this.normalTables, arg.normalTables)));
                    }
                    return Optional.empty();
                default:
                    throw new IllegalStateException("Unexpected value: " + this.type());
            }
        case BROADCAST:
            return Optional.of(new Distribution(merge(this.shardingTables, arg.shardingTables), merge(this.globalTables, arg.globalTables), merge(this.normalTables, arg.normalTables)));
        case SHARDING:
            switch(this.type()) {
                case PHY:
                    ShardingTable shardingTable = arg.shardingTables.get(0);
                    if (shardingTable.function().isAllPartitionInTargetName(this.normalTables.get(0).getDataNode().getTargetName())) {
                        return Optional.of(new Distribution(merge(this.shardingTables, arg.shardingTables), merge(this.globalTables, arg.globalTables), merge(this.normalTables, arg.normalTables)));
                    }
                    return Optional.empty();
                case BROADCAST:
                    return Optional.of(new Distribution(merge(this.shardingTables, arg.shardingTables), merge(this.globalTables, arg.globalTables), merge(this.normalTables, arg.normalTables)));
                case SHARDING:
                    ShardingTable leftShardingTable = this.shardingTables.get(0);
                    ShardingTable rightShardingTable = arg.shardingTables.get(0);
                    CustomRuleFunction leftShardingFuntion = leftShardingTable.getShardingFuntion();
                    CustomRuleFunction rightShardingFuntion = rightShardingTable.getShardingFuntion();
                    if (leftShardingFuntion.isSameDistribution(rightShardingFuntion) || isTargetPartitionJoin(leftShardingFuntion, rightShardingFuntion)) {
                        return Optional.of(new Distribution(merge(this.shardingTables, arg.shardingTables), merge(this.globalTables, arg.globalTables), merge(this.normalTables, arg.normalTables)));
                    }
                    return Optional.empty();
                default:
                    throw new IllegalStateException("Unexpected value: " + this.type());
            }
        default:
            throw new IllegalStateException("Unexpected value: " + arg.type());
    }
}

Example 2 with ShardingTable

use of io.mycat.calcite.table.ShardingTable in project Mycat2 by MyCATApache.

the class AsyncMycatDataContextImpl method getSqlMap.

public static List<PartitionGroup> getSqlMap(Map<RexNode, RexNode> constantMap, MycatView view, DrdsSqlWithParams drdsSqlWithParams, Optional<List<PartitionGroup>> hintDataMapping) {
    Distribution distribution = view.getDistribution();
    Distribution.Type type = distribution.type();
    switch(type) {
        case BROADCAST:
            {
                Map<String, Partition> builder = new HashMap<>();
                String targetName = null;
                for (GlobalTable globalTable : distribution.getGlobalTables()) {
                    if (targetName == null) {
                        int i = ThreadLocalRandom.current().nextInt(0, globalTable.getGlobalDataNode().size());
                        Partition partition = globalTable.getGlobalDataNode().get(i);
                        targetName = partition.getTargetName();
                    }
                    builder.put(globalTable.getUniqueName(), globalTable.getDataNode());
                }
                return Collections.singletonList(new PartitionGroup(targetName, builder));
            }
        case PHY:
            Map<String, Partition> builder = new HashMap<>();
            String targetName = null;
            for (GlobalTable globalTable : distribution.getGlobalTables()) {
                builder.put(globalTable.getUniqueName(), globalTable.getDataNode());
            }
            for (NormalTable normalTable : distribution.getNormalTables()) {
                if (targetName == null) {
                    targetName = normalTable.getDataNode().getTargetName();
                }
                builder.put(normalTable.getUniqueName(), normalTable.getDataNode());
            }
            return Collections.singletonList(new PartitionGroup(targetName, builder));
        case SHARDING:
            if (hintDataMapping.isPresent()) {
                return hintDataMapping.get();
            }
            ShardingTable shardingTable = distribution.getShardingTables().get(0);
            RexBuilder rexBuilder = MycatCalciteSupport.RexBuilder;
            RexNode condition = view.getCondition().orElse(MycatCalciteSupport.RexBuilder.makeLiteral(true));
            List<RexNode> inputConditions = new ArrayList<>(constantMap.size() + 1);
            inputConditions.add(condition);
            for (Map.Entry<RexNode, RexNode> rexNodeRexNodeEntry : constantMap.entrySet()) {
                inputConditions.add(rexBuilder.makeCall(SqlStdOperatorTable.EQUALS, rexNodeRexNodeEntry.getKey(), rexNodeRexNodeEntry.getValue()));
            }
            ParamHolder paramHolder = ParamHolder.CURRENT_THREAD_LOCAL.get();
            paramHolder.setData(drdsSqlWithParams.getParams(), drdsSqlWithParams.getTypeNames());
            try {
                ArrayList<RexNode> res = new ArrayList<>(inputConditions.size());
                MycatRexExecutor.INSTANCE.reduce(rexBuilder, inputConditions, res);
                condition = res.get(0);
                ValuePredicateAnalyzer predicateAnalyzer = new ValuePredicateAnalyzer(shardingTable.keyMetas(true), shardingTable.getColumns().stream().map(i -> i.getColumnName()).collect(Collectors.toList()));
                Map<QueryType, List<ValueIndexCondition>> indexConditionMap = predicateAnalyzer.translateMatch(condition);
                List<Partition> partitions = ValueIndexCondition.getPartitions(shardingTable.getShardingFuntion(), indexConditionMap, drdsSqlWithParams.getParams());
                return mapSharding(view, partitions);
            } finally {
                paramHolder.clear();
            }
        default:
            throw new IllegalStateException("Unexpected value: " + distribution.type());
    }
}

Example 3 with ShardingTable

use of io.mycat.calcite.table.ShardingTable in project Mycat2 by MyCATApache.

the class AsyncMycatDataContextImpl method mapSharding.

public static List<PartitionGroup> mapSharding(MycatView view, List<Partition> partitionList) {
    Distribution distribution = view.getDistribution();
    List<ShardingTable> shardingTableList = distribution.getShardingTables();
    ShardingTable primaryShardingTable = shardingTableList.get(0);
    CustomRuleFunction primaryShardingFunction = primaryShardingTable.getShardingFuntion();
    HashMap<String, Partition> groupTemplate = new HashMap<>();
    for (NormalTable normalTable : distribution.getNormalTables()) {
        // 可能存在错误的数据分布,但是错误的数据分布访问不到
        groupTemplate.put(normalTable.getUniqueName(), normalTable.getDataNode());
    }
    for (GlobalTable globalTable : distribution.getGlobalTables()) {
        groupTemplate.put(globalTable.getUniqueName(), globalTable.getDataNode());
    }
    if (distribution.getShardingTables().size() == 1) {
        List<PartitionGroup> res = new ArrayList<>(partitionList.size());
        for (Partition partition : partitionList) {
            HashMap<String, Partition> map = new HashMap<>(groupTemplate);
            map.put(primaryShardingTable.getUniqueName(), partition);
            res.add(new PartitionGroup(partition.getTargetName(), map));
        }
        return res;
    } else {
        List<ShardingTable> joinShardingTables = shardingTableList.subList(1, shardingTableList.size());
        List<PartitionGroup> res = new ArrayList<>(partitionList.size());
        for (Partition primaryPartition : partitionList) {
            HashMap<String, Partition> map = new HashMap<>(groupTemplate);
            map.put(primaryShardingTable.getUniqueName(), primaryPartition);
            for (ShardingTable joinShardingTable : joinShardingTables) {
                CustomRuleFunction joinFunction = joinShardingTable.function();
                if (primaryShardingFunction.isSameDistribution(joinFunction)) {
                    Partition joinPartition = joinFunction.getPartition(primaryShardingFunction.indexOf(primaryPartition));
                    map.put(joinShardingTable.getUniqueName(), joinPartition);
                } else if (primaryShardingFunction.isSameTargetFunctionDistribution(joinFunction)) {
                    List<Partition> joinPartitions = joinShardingTable.getPartitionsByTargetName(primaryPartition.getTargetName());
                    if (joinPartitions.size() != 1) {
                        throw new IllegalArgumentException("wrong partition " + joinPartitions + " in " + view);
                    }
                    map.put(joinShardingTable.getUniqueName(), joinPartitions.get(0));
                }
            }
            res.add(new PartitionGroup(primaryPartition.getTargetName(), map));
        }
        return res;
    }
}

Example 4 with ShardingTable

use of io.mycat.calcite.table.ShardingTable in project Mycat2 by MyCATApache.

the class ColocatedPlanner method checkColocatedPushDown.

public static Optional<PartitionGroup> checkColocatedPushDown(MycatDataContext context, Plan plan, DrdsSqlWithParams drdsSqlWithParams) {
    CodeExecuterContext codeExecuterContext = plan.getCodeExecuterContext();
    AsyncMycatDataContextImpl.SqlMycatDataContextImpl sqlMycatDataContext = new AsyncMycatDataContextImpl.SqlMycatDataContextImpl(context, codeExecuterContext, drdsSqlWithParams);
    Map<String, MycatRelDatasourceSourceInfo> relContext = codeExecuterContext.getRelContext();
    List<List<PartitionGroup>> lists = new ArrayList<>();
    for (MycatRelDatasourceSourceInfo mycatRelDatasourceSourceInfo : relContext.values()) {
        if (mycatRelDatasourceSourceInfo.getRelNode() instanceof MycatView) {
            MycatView mycatView = (MycatView) mycatRelDatasourceSourceInfo.getRelNode();
            if (mycatView.getDistribution().type() == Distribution.Type.SHARDING) {
                for (ShardingTable shardingTable : mycatView.getDistribution().getShardingTables()) {
                    if (shardingTable instanceof ShardingIndexTable) {
                        return Optional.empty();
                    }
                }
            }
            lists.add(sqlMycatDataContext.getPartition(mycatView.getDigest()).orElse(Collections.emptyList()));
        } else {
            return Optional.empty();
        }
    }
    // Colocated Push Down
    PartitionGroup result = null;
    for (List<PartitionGroup> list : lists) {
        if (list.size() == 1) {
            PartitionGroup each = list.get(0);
            if (result == null) {
                result = new PartitionGroup(each.getTargetName(), new HashMap<>(each.getMap()));
            } else if (result.getTargetName().equals(each.getTargetName())) {
                Map<String, Partition> eachMap = each.getMap();
                Map<String, Partition> resMap = result.getMap();
                for (Map.Entry<String, Partition> entry : eachMap.entrySet()) {
                    if (resMap.containsKey(entry.getKey())) {
                        // 已经存在的分区不一致,所以不匹配
                        Partition existed = resMap.get(entry.getKey());
                        if (!existed.getUniqueName().equals(entry.getValue().getUniqueName())) {
                            return Optional.empty();
                        }
                    } else {
                        resMap.put(entry.getKey(), entry.getValue());
                    }
                }
            } else {
                return Optional.empty();
            }
        } else {
            return Optional.empty();
        }
    }
    return Optional.ofNullable(result);
}

Example 5 with ShardingTable

use of io.mycat.calcite.table.ShardingTable in project Mycat2 by MyCATApache.

the class EnumTest method testHashMM.

@Test
public void testHashMM() {
    ShardingTableConfig mainSharding = new ShardingTableConfig();
    mainSharding.setCreateTableSQL("CREATE TABLE db1.`sharding` (\n" + "  `id` bigint NOT NULL AUTO_INCREMENT,\n" + "  `user_id` varchar(100) DEFAULT NULL,\n" + "  `traveldate` date DEFAULT NULL,\n" + "  `fee` decimal(10,0) DEFAULT NULL,\n" + "  `days` int DEFAULT NULL,\n" + "  `blob` longblob,\n" + "  PRIMARY KEY (`id`),\n" + "  KEY `id` (`id`)\n" + ") ENGINE=InnoDB  DEFAULT CHARSET=utf8" + " dbpartition by mod_hash(id) tbpartition by MM(traveldate) tbpartitions 2 dbpartitions 4;");
    mainSharding.setFunction(ShardingFunction.builder().properties(JsonUtil.from("{\n" + "\t\t\t\t\t\"dbNum\":\"2\",\n" + "\t\t\t\t\t\"mappingFormat\":\"c${targetIndex}/db1_${dbIndex}/sharding_${tableIndex}\",\n" + "\t\t\t\t\t\"tableNum\":\"4\",\n" + "\t\t\t\t\t\"tableMethod\":\"MM(traveldate)\",\n" + "\t\t\t\t\t\"storeNum\":1,\n" + "\t\t\t\t\t\"dbMethod\":\"hash(id)\"\n" + "\t\t\t\t}", Map.class)).build());
    MetadataManager metadataManager = getMetadataManager(mainSharding);
    ShardingTable tableHandler = (ShardingTable) metadataManager.getTable("db1", "sharding");
    CustomRuleFunction shardingFuntion = tableHandler.getShardingFuntion();
    LocalDate start = LocalDate.of(2021, 11, 8);
    List<Partition> calculate = shardingFuntion.calculate(Collections.singletonMap("traveldate", (new RangeVariable("traveldate", RangeVariableType.EQUAL, start))));
    Assert.assertEquals(2, calculate.size());
    for (Partition partition : calculate) {
        Assert.assertEquals(start.getMonthValue() % 4, (int) partition.getTableIndex());
    }
    {
        LocalDate end = LocalDate.of(2021, 11, 9);
        List<Partition> calculate2 = shardingFuntion.calculate(Collections.singletonMap("traveldate", (new RangeVariable("traveldate", RangeVariableType.RANGE, start, end))));
        Assert.assertEquals(calculate, calculate2);
    }
    {
        LocalDate end = LocalDate.of(2021, 12, 9);
        List<Partition> calculate2 = shardingFuntion.calculate(Collections.singletonMap("traveldate", (new RangeVariable("traveldate", RangeVariableType.RANGE, start, end))));
        HashSet<Integer> set = new HashSet<>();
        for (int i = 11; i <= 12; i++) {
            set.add(i % 4);
        }
        Assert.assertEquals(true, calculate2.stream().allMatch(i -> set.contains(i.getTableIndex())));
        System.out.println();
    }
    {
        LocalDate end = LocalDate.of(2022, 1, 9);
        List<Partition> calculate2 = shardingFuntion.calculate(Collections.singletonMap("traveldate", (new RangeVariable("traveldate", RangeVariableType.RANGE, start, end))));
        HashSet<Integer> set = new HashSet<>();
        for (Integer integer : Arrays.asList(11, 12, 1)) {
            set.add(integer % 4);
        }
        Assert.assertEquals(true, calculate2.stream().allMatch(i -> set.contains(i.getTableIndex())));
        System.out.println();
    }
}