Examples with ShardingConfig org.sagacity.sqltoy.config.model.ShardingConfig used on opensource projects

Example 1 with ShardingConfig

use of org.sagacity.sqltoy.config.model.ShardingConfig in project sagacity-sqltoy by chenrenfei.

the class ParallelUtils method execute.

/**
 * @todo 将集合进行根据sharding字段的值提取sharding策略并按照策略将集合分组，然后并行执行
 * @param sqlToyContext
 * @param entities
 * @param dataSource
 * @param handler
 * @return
 * @throws Exception
 */
public static List execute(final SqlToyContext sqlToyContext, List entities, boolean wrapIdValue, DataSource dataSource, ParallelCallbackHandler handler) throws Exception {
    // 获取对象的媒体信息
    EntityMeta entityMeta = sqlToyContext.getEntityMeta(entities.get(0).getClass());
    // 主键值需要提前按照主键策略赋予(sequence 和assign模式的不会实际执行赋值)
    if (wrapIdValue) {
        ShardingUtils.assignPKs(sqlToyContext, entityMeta, entities);
    }
    // 将批量集合数据按sharding策略处理后的库和表组成的key分组
    Collection<ShardingGroupModel> shardingGroups = ShardingUtils.groupShardings(sqlToyContext, entities, entityMeta, dataSource);
    // 单分组直接执行
    if (shardingGroups.size() == 1) {
        return handler.execute(sqlToyContext, shardingGroups.iterator().next());
    }
    // 开始多线程并行执行
    ShardingConfig shardingConfig = entityMeta.getShardingConfig();
    List results = new ArrayList();
    // 并行线程数量
    int threads = shardingGroups.size();
    // 如果额外策略配置了线程数量,则按照指定的线程数量执行
    if (threads > shardingConfig.getMaxConcurrents() && shardingConfig.getMaxConcurrents() > 1)
        threads = shardingConfig.getMaxConcurrents();
    ExecutorService pool = Executors.newFixedThreadPool(threads);
    List<Future<ShardingResult>> futureResult = new ArrayList<Future<ShardingResult>>();
    for (final ShardingGroupModel group : shardingGroups) {
        Future<ShardingResult> future = pool.submit(new DialectExecutor(sqlToyContext, group, handler));
        futureResult.add(future);
    }
    pool.shutdown();
    // 设置最大等待时长
    if (shardingConfig.getMaxWaitSeconds() > 0)
        pool.awaitTermination(shardingConfig.getMaxWaitSeconds(), TimeUnit.SECONDS);
    // 提取各个线程返回的结果进行合并
    try {
        for (Future<ShardingResult> future : futureResult) {
            ShardingResult item = future.get();
            if (item != null && item.getRows() != null && !item.getRows().isEmpty())
                results.addAll(item.getRows());
        }
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    } finally {
        pool.shutdownNow();
    }
    return results;
}

Example 2 with ShardingConfig

use of org.sagacity.sqltoy.config.model.ShardingConfig in project sagacity-sqltoy by chenrenfei.

the class ShardingUtils method getSharding.

/**
 * @todo 单个对象sharding策略处理,适用于load、save、update、delete单对象操作
 * @param sqlToyContext
 * @param entity
 * @param wrapIdValue
 * @param dataSource
 * @return
 * @throws Exception
 */
public static ShardingModel getSharding(SqlToyContext sqlToyContext, Serializable entity, boolean wrapIdValue, DataSource dataSource) throws Exception {
    ShardingModel shardingModel = new ShardingModel();
    shardingModel.setDataSource(dataSource);
    EntityMeta entityMeta = sqlToyContext.getEntityMeta(entity.getClass());
    // 主键值需要提前按照主键策略赋予(sequence 和assign模式的不会实际执行赋值)
    if (wrapIdValue)
        assignPK(sqlToyContext, entityMeta, entity);
    ShardingConfig shardingConfig = entityMeta.getShardingConfig();
    if (shardingConfig == null)
        return shardingModel;
    ShardingStrategy shardingStrategy;
    ShardingStrategyConfig strategyConfig;
    // 分库策略处理
    if (shardingConfig.getShardingDBStrategy() != null) {
        strategyConfig = shardingConfig.getShardingDBStrategy();
        shardingStrategy = sqlToyContext.getShardingStrategy(strategyConfig.getName());
        if (shardingStrategy == null)
            throw new Exception("POJO 对象:" + entity.getClass().getName() + " Sharding DB Strategy:" + strategyConfig.getName() + " 未定义,请检查!");
        IgnoreCaseLinkedMap<String, Object> valueMap = hashParams(strategyConfig.getAliasNames(), BeanUtil.reflectBeanToAry(entity, strategyConfig.getFields(), null, null));
        ShardingDBModel dbModel = shardingStrategy.getShardingDB(sqlToyContext, entity.getClass(), entityMeta.getSchemaTable(), strategyConfig.getDecisionType(), valueMap);
        shardingModel.setDataSourceName(dbModel.getDataSourceName());
        if (dbModel.getDataSource() == null)
            shardingModel.setDataSource(sqlToyContext.getDataSource(dbModel.getDataSourceName()));
        else
            shardingModel.setDataSource(dbModel.getDataSource());
    }
    // 分表策略
    if (shardingConfig.getShardingTableStrategy() != null) {
        strategyConfig = shardingConfig.getShardingTableStrategy();
        shardingStrategy = sqlToyContext.getShardingStrategy(strategyConfig.getName());
        if (shardingStrategy == null)
            throw new Exception("POJO 对象:" + entity.getClass().getName() + " Sharding Table Strategy:" + strategyConfig.getName() + " 未定义,请检查!");
        IgnoreCaseLinkedMap<String, Object> valueMap = hashParams(strategyConfig.getAliasNames(), BeanUtil.reflectBeanToAry(entity, strategyConfig.getFields(), null, null));
        String tableName = shardingStrategy.getShardingTable(sqlToyContext, entity.getClass(), entityMeta.getSchemaTable(), strategyConfig.getDecisionType(), valueMap);
        if (StringUtil.isNotBlank(tableName))
            shardingModel.setTableName(tableName);
    }
    return shardingModel;
}

Example 3 with ShardingConfig

use of org.sagacity.sqltoy.config.model.ShardingConfig in project sagacity-sqltoy by chenrenfei.

the class ShardingUtils method groupShardings.

/**
 * @todo 批量sharding策略处理
 * @param sqlToyContext
 * @param entities
 * @param entityMeta
 * @param dataSource
 * @return
 * @throws Exception
 */
public static Collection<ShardingGroupModel> groupShardings(SqlToyContext sqlToyContext, List<?> entities, EntityMeta entityMeta, DataSource dataSource) throws Exception {
    ShardingConfig shardingConfig = entityMeta.getShardingConfig();
    ShardingModel shardingModel = null;
    // 没有sharding配置，则作为单个分组返回
    if (shardingConfig == null) {
        Collection<ShardingGroupModel> result = new ArrayList<ShardingGroupModel>();
        ShardingGroupModel model = new ShardingGroupModel();
        shardingModel = new ShardingModel();
        shardingModel.setDataSource(dataSource);
        model.setShardingModel(shardingModel);
        model.setEntities(entities);
        result.add(model);
        return result;
    }
    Class entityClass = entities.get(0).getClass();
    String entityTable = entityMeta.getSchemaTable();
    // 分库
    boolean hasDB = false;
    ShardingStrategy dbStrategy = null;
    List<Object[]> shardingDBValues = null;
    ShardingStrategyConfig dbConfig = shardingConfig.getShardingDBStrategy();
    if (dbConfig != null) {
        hasDB = true;
        dbStrategy = sqlToyContext.getShardingStrategy(dbConfig.getName());
        if (dbStrategy == null)
            throw new Exception("POJO 对象:" + entityClass.getName() + " Sharding DB Strategy:" + dbConfig.getName() + " 未定义,请检查!");
        shardingDBValues = BeanUtil.reflectBeansToInnerAry(entities, dbConfig.getFields(), null, null, false, 0);
    }
    // 分表
    boolean hasTable = false;
    ShardingStrategy tableStrategy = null;
    ShardingStrategyConfig tableConfig = shardingConfig.getShardingTableStrategy();
    List<Object[]> shardingTableValues = null;
    if (tableConfig != null) {
        hasTable = true;
        tableStrategy = sqlToyContext.getShardingStrategy(tableConfig.getName());
        if (tableStrategy == null)
            throw new Exception("POJO 对象:" + entityClass.getName() + " Sharding Table Strategy:" + tableConfig.getName() + " 未定义,请检查!");
        shardingTableValues = BeanUtil.reflectBeansToInnerAry(entities, tableConfig.getFields(), null, null, false, 0);
    }
    Map<String, ShardingGroupModel> shardingGroupMaps = new HashMap<String, ShardingGroupModel>();
    IgnoreCaseLinkedMap<String, Object> valueMap;
    ShardingDBModel shardingDBModel = null;
    // 数据分组key(dataSourceName+tableName)
    String dataGroupKey;
    String tableName = null;
    String dataSourceName = null;
    for (int i = 0; i < entities.size(); i++) {
        // 分库
        if (hasDB) {
            valueMap = hashParams(dbConfig.getAliasNames(), shardingDBValues.get(i));
            shardingDBModel = dbStrategy.getShardingDB(sqlToyContext, entityClass, entityTable, dbConfig.getDecisionType(), valueMap);
            dataSourceName = shardingDBModel.getDataSourceName();
        }
        // 分表
        if (hasTable) {
            valueMap = hashParams(tableConfig.getAliasNames(), shardingTableValues.get(i));
            tableName = tableStrategy.getShardingTable(sqlToyContext, entityClass, entityTable, tableConfig.getDecisionType(), valueMap);
        }
        dataGroupKey = dataSourceName + tableName;
        // 归并到相同分组
        if (shardingGroupMaps.containsKey(dataGroupKey)) {
            shardingGroupMaps.get(dataGroupKey).getEntities().add(entities.get(i));
        } else {
            // 不同分组
            ShardingGroupModel groupModel = new ShardingGroupModel();
            groupModel.setKey(dataGroupKey);
            // 创建数据分组集合
            List items = new ArrayList();
            items.add(entities.get(i));
            groupModel.setEntities(items);
            shardingModel = new ShardingModel();
            // 分库,设置分组对应的数据库
            if (hasDB) {
                shardingModel.setDataSourceName(dataSourceName);
                if (shardingDBModel.getDataSource() == null)
                    shardingModel.setDataSource(sqlToyContext.getDataSource(shardingDBModel.getDataSourceName()));
                else
                    shardingModel.setDataSource(shardingDBModel.getDataSource());
            } else
                shardingModel.setDataSource(dataSource);
            // 分表,设置表名
            if (hasTable && StringUtil.isNotBlank(tableName))
                shardingModel.setTableName(tableName);
            groupModel.setShardingModel(shardingModel);
            shardingGroupMaps.put(dataGroupKey, groupModel);
        }
    }
    return shardingGroupMaps.values();
}

Example 4 with ShardingConfig

use of org.sagacity.sqltoy.config.model.ShardingConfig in project sagacity-sqltoy by chenrenfei.

the class EntityManager method parseSharding.

/**
 * @todo 解析分库分表策略
 * @param entityMeta
 * @param entityClass
 */
private void parseSharding(EntityMeta entityMeta, Class entityClass) {
    Class classType = entityClass;
    Sharding sharding = null;
    // 增加递归对父类检测
    while (classType != null && !classType.equals(Object.class)) {
        sharding = (Sharding) classType.getAnnotation(Sharding.class);
        if (sharding != null) {
            break;
        }
        classType = classType.getSuperclass();
    }
    // 不存在分库策略
    if (sharding == null) {
        return;
    }
    // 分库策略
    ShardingConfig shardingConfig = new ShardingConfig();
    // 最大并行数量
    shardingConfig.setMaxConcurrents(sharding.maxConcurrents());
    // 最大执行时长(秒)
    shardingConfig.setMaxWaitSeconds(sharding.maxWaitSeconds());
    // 异常处理策略(是否全局回滚)
    shardingConfig.setGlobalRollback(sharding.is_global_rollback());
    Strategy shardingDB = sharding.db();
    String strategy = shardingDB.name();
    // 分库策略
    if (StringUtil.isNotBlank(strategy)) {
        ShardingStrategyConfig config = new ShardingStrategyConfig(0);
        config.setFields(shardingDB.fields());
        // 别名,如果没有设置则将fields作为默认别名,别名的目的在于共用sharding策略中的参数名称
        String[] aliasNames = new String[shardingDB.fields().length];
        System.arraycopy(shardingDB.fields(), 0, aliasNames, 0, aliasNames.length);
        if (shardingDB.aliasNames() != null) {
            System.arraycopy(shardingDB.aliasNames(), 0, aliasNames, 0, shardingDB.aliasNames().length);
        }
        config.setAliasNames(aliasNames);
        config.setDecisionType(shardingDB.decisionType());
        config.setStrategy(strategy);
        shardingConfig.setShardingDBStrategy(config);
    }
    // 分表策略
    Strategy shardingTable = sharding.table();
    strategy = shardingTable.name();
    if (StringUtil.isNotBlank(strategy)) {
        ShardingStrategyConfig config = new ShardingStrategyConfig(1);
        config.setFields(shardingTable.fields());
        // 别名,如果没有设置则将fields作为默认别名,别名的目的在于共用sharding策略中的参数名称
        String[] aliasNames = new String[shardingTable.fields().length];
        System.arraycopy(shardingTable.fields(), 0, aliasNames, 0, aliasNames.length);
        if (shardingTable.aliasNames() != null) {
            System.arraycopy(shardingTable.aliasNames(), 0, aliasNames, 0, shardingTable.aliasNames().length);
        }
        config.setTables(new String[] { entityMeta.getTableName() });
        config.setAliasNames(aliasNames);
        config.setDecisionType(shardingDB.decisionType());
        config.setStrategy(strategy);
        shardingConfig.setShardingTableStrategy(config);
    }
    // 必须有一个策略是存在的
    if (shardingConfig.getShardingDBStrategy() != null || shardingConfig.getShardingTableStrategy() != null) {
        entityMeta.setShardingConfig(shardingConfig);
    }
}

Example 5 with ShardingConfig

use of org.sagacity.sqltoy.config.model.ShardingConfig in project sagacity-sqltoy by chenrenfei.

the class ShardingUtils method getSharding.

/**
 * @todo 单个对象sharding策略处理,适用于load、save、update、delete单对象操作
 * @param sqlToyContext
 * @param entity
 * @param wrapIdValue
 * @param dataSource
 * @return
 * @throws Exception
 */
public static ShardingModel getSharding(SqlToyContext sqlToyContext, Serializable entity, boolean wrapIdValue, DataSource dataSource) throws Exception {
    ShardingModel shardingModel = new ShardingModel();
    shardingModel.setDataSource(dataSource);
    EntityMeta entityMeta = sqlToyContext.getEntityMeta(entity.getClass());
    shardingModel.setTableName(entityMeta.getTableName());
    // 主键值需要提前按照主键策略赋予(sequence 和assign模式的不会实际执行赋值)
    if (wrapIdValue) {
        assignPK(sqlToyContext, entityMeta, entity);
    }
    ShardingConfig shardingConfig = entityMeta.getShardingConfig();
    if (shardingConfig == null) {
        return shardingModel;
    }
    ShardingStrategy shardingStrategy;
    ShardingStrategyConfig strategyConfig;
    // 分库策略处理
    if (shardingConfig.getShardingDBStrategy() != null) {
        strategyConfig = shardingConfig.getShardingDBStrategy();
        shardingStrategy = sqlToyContext.getShardingStrategy(strategyConfig.getStrategy());
        if (shardingStrategy == null) {
            throw new IllegalArgumentException("POJO 对象:" + entity.getClass().getName() + " Sharding DB Strategy:" + strategyConfig.getStrategy() + " 未定义,请检查!");
        }
        IgnoreCaseLinkedMap<String, Object> valueMap = hashParams(strategyConfig.getAliasNames(), BeanUtil.reflectBeanToAry(entity, strategyConfig.getFields()));
        ShardingDBModel dbModel = shardingStrategy.getShardingDB(sqlToyContext, entity.getClass(), entityMeta.getTableName(), strategyConfig.getDecisionType(), valueMap);
        shardingModel.setDataSourceName(dbModel.getDataSourceName());
        if (dbModel.getDataSource() == null) {
            shardingModel.setDataSource(sqlToyContext.getDataSourceBean(dbModel.getDataSourceName()));
        } else {
            shardingModel.setDataSource(dbModel.getDataSource());
        }
    }
    // 分表策略
    if (shardingConfig.getShardingTableStrategy() != null) {
        strategyConfig = shardingConfig.getShardingTableStrategy();
        shardingStrategy = sqlToyContext.getShardingStrategy(strategyConfig.getStrategy());
        if (shardingStrategy == null) {
            throw new IllegalArgumentException("POJO 对象:" + entity.getClass().getName() + " Sharding Table Strategy:" + strategyConfig.getStrategy() + " 未定义,请检查!");
        }
        IgnoreCaseLinkedMap<String, Object> valueMap = hashParams(strategyConfig.getAliasNames(), BeanUtil.reflectBeanToAry(entity, strategyConfig.getFields()));
        String tableName = shardingStrategy.getShardingTable(sqlToyContext, entity.getClass(), entityMeta.getTableName(), strategyConfig.getDecisionType(), valueMap);
        if (StringUtil.isNotBlank(tableName)) {
            shardingModel.setTableName(tableName);
        }
    }
    return shardingModel;
}