Examples with RegionPlan org.apache.hadoop.hbase.master.RegionPlan used on opensource projects

Example 21 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class RSGroupAdminServer method balanceRSGroup.

@Override
public boolean balanceRSGroup(String groupName) throws IOException {
    ServerManager serverManager = master.getServerManager();
    AssignmentManager assignmentManager = master.getAssignmentManager();
    LoadBalancer balancer = master.getLoadBalancer();
    boolean balancerRan;
    synchronized (balancer) {
        if (master.getMasterCoprocessorHost() != null) {
            master.getMasterCoprocessorHost().preBalanceRSGroup(groupName);
        }
        if (getRSGroupInfo(groupName) == null) {
            throw new ConstraintException("RSGroup does not exist: " + groupName);
        }
        // Only allow one balance run at at time.
        Map<String, RegionState> groupRIT = rsGroupGetRegionsInTransition(groupName);
        if (groupRIT.size() > 0) {
            LOG.debug("Not running balancer because " + groupRIT.size() + " region(s) in transition: " + StringUtils.abbreviate(master.getAssignmentManager().getRegionStates().getRegionsInTransition().toString(), 256));
            return false;
        }
        if (serverManager.areDeadServersInProgress()) {
            LOG.debug("Not running balancer because processing dead regionserver(s): " + serverManager.getDeadServers());
            return false;
        }
        //We balance per group instead of per table
        List<RegionPlan> plans = new ArrayList<>();
        for (Map.Entry<TableName, Map<ServerName, List<HRegionInfo>>> tableMap : getRSGroupAssignmentsByTable(groupName).entrySet()) {
            LOG.info("Creating partial plan for table " + tableMap.getKey() + ": " + tableMap.getValue());
            List<RegionPlan> partialPlans = balancer.balanceCluster(tableMap.getValue());
            LOG.info("Partial plan for table " + tableMap.getKey() + ": " + partialPlans);
            if (partialPlans != null) {
                plans.addAll(partialPlans);
            }
        }
        long startTime = System.currentTimeMillis();
        balancerRan = plans != null;
        if (plans != null && !plans.isEmpty()) {
            LOG.info("RSGroup balance " + groupName + " starting with plan count: " + plans.size());
            for (RegionPlan plan : plans) {
                LOG.info("balance " + plan);
                assignmentManager.balance(plan);
            }
            LOG.info("RSGroup balance " + groupName + " completed after " + (System.currentTimeMillis() - startTime) + " seconds");
        }
        if (master.getMasterCoprocessorHost() != null) {
            master.getMasterCoprocessorHost().postBalanceRSGroup(groupName, balancerRan);
        }
    }
    return balancerRan;
}

Example 22 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class RSGroupBasedLoadBalancer method balanceCluster.

@Override
public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) throws HBaseIOException {
    if (!isOnline()) {
        throw new ConstraintException(RSGroupInfoManager.RSGROUP_TABLE_NAME + " is not online, unable to perform balance");
    }
    Map<ServerName, List<HRegionInfo>> correctedState = correctAssignments(clusterState);
    List<RegionPlan> regionPlans = new ArrayList<>();
    List<HRegionInfo> misplacedRegions = correctedState.get(LoadBalancer.BOGUS_SERVER_NAME);
    for (HRegionInfo regionInfo : misplacedRegions) {
        regionPlans.add(new RegionPlan(regionInfo, null, null));
    }
    try {
        List<RSGroupInfo> rsgi = rsGroupInfoManager.listRSGroups();
        for (RSGroupInfo info : rsgi) {
            Map<ServerName, List<HRegionInfo>> groupClusterState = new HashMap<>();
            Map<TableName, Map<ServerName, List<HRegionInfo>>> groupClusterLoad = new HashMap<>();
            for (Address sName : info.getServers()) {
                for (ServerName curr : clusterState.keySet()) {
                    if (curr.getAddress().equals(sName)) {
                        groupClusterState.put(curr, correctedState.get(curr));
                    }
                }
            }
            groupClusterLoad.put(TableName.valueOf(HConstants.ENSEMBLE_TABLE_NAME), groupClusterState);
            this.internalBalancer.setClusterLoad(groupClusterLoad);
            List<RegionPlan> groupPlans = this.internalBalancer.balanceCluster(groupClusterState);
            if (groupPlans != null) {
                regionPlans.addAll(groupPlans);
            }
        }
    } catch (IOException exp) {
        LOG.warn("Exception while balancing cluster.", exp);
        regionPlans.clear();
    }
    return regionPlans;
}

Example 23 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class BalancerTestBase method reconcile.

/**
   * This assumes the RegionPlan HSI instances are the same ones in the map, so
   * actually no need to even pass in the map, but I think it's clearer.
   *
   * @param list
   * @param plans
   * @return
   */
protected List<ServerAndLoad> reconcile(List<ServerAndLoad> list, List<RegionPlan> plans, Map<ServerName, List<HRegionInfo>> servers) {
    List<ServerAndLoad> result = new ArrayList<>(list.size());
    Map<ServerName, ServerAndLoad> map = new HashMap<>(list.size());
    for (ServerAndLoad sl : list) {
        map.put(sl.getServerName(), sl);
    }
    if (plans != null) {
        for (RegionPlan plan : plans) {
            ServerName source = plan.getSource();
            updateLoad(map, source, -1);
            ServerName destination = plan.getDestination();
            updateLoad(map, destination, +1);
            servers.get(source).remove(plan.getRegionInfo());
            servers.get(destination).add(plan.getRegionInfo());
        }
    }
    result.clear();
    result.addAll(map.values());
    return result;
}

Example 24 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class RSGroupBasedLoadBalancer method balanceCluster.

/**
 * Balance by RSGroup.
 */
@Override
public synchronized List<RegionPlan> balanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) throws IOException {
    if (!isOnline()) {
        throw new ConstraintException(RSGroupInfoManager.class.getSimpleName() + " is not online, unable to perform balance");
    }
    // Calculate correct assignments and a list of RegionPlan for mis-placed regions
    Pair<Map<TableName, Map<ServerName, List<RegionInfo>>>, List<RegionPlan>> correctedStateAndRegionPlans = correctAssignments(loadOfAllTable);
    Map<TableName, Map<ServerName, List<RegionInfo>>> correctedLoadOfAllTable = correctedStateAndRegionPlans.getFirst();
    List<RegionPlan> regionPlans = correctedStateAndRegionPlans.getSecond();
    RSGroupInfo defaultInfo = rsGroupInfoManager.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
    // into the movement list
    try {
        // For each rsgroup
        for (RSGroupInfo rsgroup : rsGroupInfoManager.listRSGroups()) {
            LOG.debug("Balancing RSGroup={}", rsgroup.getName());
            Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfTablesInGroup = new HashMap<>();
            for (Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> entry : correctedLoadOfAllTable.entrySet()) {
                TableName tableName = entry.getKey();
                RSGroupInfo targetRSGInfo = RSGroupUtil.getRSGroupInfo(masterServices, rsGroupInfoManager, tableName).orElse(defaultInfo);
                if (targetRSGInfo.getName().equals(rsgroup.getName())) {
                    loadOfTablesInGroup.put(tableName, entry.getValue());
                }
            }
            List<RegionPlan> groupPlans = null;
            if (!loadOfTablesInGroup.isEmpty()) {
                LOG.info("Start Generate Balance plan for group: " + rsgroup.getName());
                groupPlans = this.internalBalancer.balanceCluster(loadOfTablesInGroup);
            }
            if (groupPlans != null) {
                regionPlans.addAll(groupPlans);
            }
        }
    } catch (IOException exp) {
        LOG.warn("Exception while balancing cluster.", exp);
        regionPlans.clear();
    }
    // Return the whole movement list
    return regionPlans;
}

Example 25 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class RSGroupBasedLoadBalancer method correctAssignments.

private Pair<Map<TableName, Map<ServerName, List<RegionInfo>>>, List<RegionPlan>> correctAssignments(Map<TableName, Map<ServerName, List<RegionInfo>>> existingAssignments) throws IOException {
    // To return
    Map<TableName, Map<ServerName, List<RegionInfo>>> correctAssignments = new HashMap<>();
    List<RegionPlan> regionPlansForMisplacedRegions = new ArrayList<>();
    RSGroupInfo defaultInfo = rsGroupInfoManager.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
    for (Map.Entry<TableName, Map<ServerName, List<RegionInfo>>> assignments : existingAssignments.entrySet()) {
        TableName tableName = assignments.getKey();
        Map<ServerName, List<RegionInfo>> clusterLoad = assignments.getValue();
        RSGroupInfo targetRSGInfo = null;
        Map<ServerName, List<RegionInfo>> correctServerRegion = new TreeMap<>();
        try {
            targetRSGInfo = RSGroupUtil.getRSGroupInfo(masterServices, rsGroupInfoManager, tableName).orElse(defaultInfo);
        } catch (IOException exp) {
            LOG.debug("RSGroup information null for region of table " + tableName, exp);
        }
        for (Map.Entry<ServerName, List<RegionInfo>> serverRegionMap : clusterLoad.entrySet()) {
            ServerName currentHostServer = serverRegionMap.getKey();
            List<RegionInfo> regionInfoList = serverRegionMap.getValue();
            if (targetRSGInfo == null || !targetRSGInfo.containsServer(currentHostServer.getAddress())) {
                regionInfoList.forEach(regionInfo -> {
                    regionPlansForMisplacedRegions.add(new RegionPlan(regionInfo, currentHostServer, null));
                });
            } else {
                correctServerRegion.put(currentHostServer, regionInfoList);
            }
        }
        correctAssignments.put(tableName, correctServerRegion);
    }
    // Return correct assignments and region movement plan for mis-placed regions together
    return new Pair<Map<TableName, Map<ServerName, List<RegionInfo>>>, List<RegionPlan>>(correctAssignments, regionPlansForMisplacedRegions);
}