Search in sources :

Example 6 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class AssignmentVerificationReport method fillUpDispersion.

/**
 * Use this to project the dispersion scores
 * @param tableName
 * @param snapshot
 * @param newPlan
 */
public void fillUpDispersion(TableName tableName, SnapshotOfRegionAssignmentFromMeta snapshot, FavoredNodesPlan newPlan) {
    // Set the table name
    this.tableName = tableName;
    // Get all the regions for this table
    List<RegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName);
    // Get the total region num for the current table
    this.totalRegions = regionInfoList.size();
    FavoredNodesPlan plan = null;
    if (newPlan == null) {
        plan = snapshot.getExistingAssignmentPlan();
    } else {
        plan = newPlan;
    }
    // Get the region to region server mapping
    Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<>();
    Map<ServerName, Set<ServerName>> primaryToSecTerRSMap = new HashMap<>();
    // Also keep tracker of the most loaded and least loaded region servers
    for (RegionInfo region : regionInfoList) {
        try {
            // Get the favored nodes from the assignment plan and verify it.
            List<ServerName> favoredNodes = plan.getFavoredNodes(region);
            if (favoredNodes == null || favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
                regionsWithoutValidFavoredNodes.add(region);
                continue;
            }
            // Get the primary, secondary and tertiary region server
            ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal());
            ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal());
            ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal());
            // Update the primary rs to its region set map
            Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS);
            if (regionCounter == null) {
                regionCounter = Integer.valueOf(0);
            }
            regionCounter = regionCounter.intValue() + 1;
            primaryRSToRegionCounterMap.put(primaryRS, regionCounter);
            // Update the primary rs to secondary and tertiary rs map
            Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS);
            if (secAndTerSet == null) {
                secAndTerSet = new HashSet<>();
            }
            secAndTerSet.add(secondaryRS);
            secAndTerSet.add(tertiaryRS);
            primaryToSecTerRSMap.put(primaryRS, secAndTerSet);
        } catch (Exception e) {
            LOG.error("Cannot verify the region assignment for region " + ((region == null) ? " null " : region.getRegionNameAsString()) + "because of " + e);
        }
    }
    float dispersionScoreSummary = 0;
    float dispersionNumSummary = 0;
    // Calculate the secondary score for each primary region server
    for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) {
        ServerName primaryRS = entry.getKey();
        Integer regionsOnPrimary = entry.getValue();
        // Process the dispersion number and score
        float dispersionScore = 0;
        int dispersionNum = 0;
        if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) {
            dispersionNum = primaryToSecTerRSMap.get(primaryRS).size();
            dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2);
        }
        // Update the max dispersion num
        if (dispersionNum > this.maxDispersionNum) {
            this.maxDispersionNumServerSet.clear();
            this.maxDispersionNumServerSet.add(primaryRS);
            this.maxDispersionNum = dispersionNum;
        } else if (dispersionNum == this.maxDispersionNum) {
            this.maxDispersionNumServerSet.add(primaryRS);
        }
        // Update the min dispersion score
        if (dispersionScore < this.minDispersionScore) {
            this.minDispersionScoreServerSet.clear();
            this.minDispersionScoreServerSet.add(primaryRS);
            this.minDispersionScore = dispersionScore;
        } else if (dispersionScore == this.minDispersionScore) {
            this.minDispersionScoreServerSet.add(primaryRS);
        }
        // Update the min dispersion num
        if (dispersionNum < this.minDispersionNum) {
            this.minDispersionNumServerSet.clear();
            this.minDispersionNumServerSet.add(primaryRS);
            this.minDispersionNum = dispersionNum;
        } else if (dispersionNum == this.minDispersionNum) {
            this.minDispersionNumServerSet.add(primaryRS);
        }
        dispersionScoreSummary += dispersionScore;
        dispersionNumSummary += dispersionNum;
    }
    // Update the avg dispersion score
    if (primaryRSToRegionCounterMap.keySet().size() != 0) {
        this.avgDispersionScore = dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size();
        this.avgDispersionNum = dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size();
    }
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) FavoredNodesPlan(org.apache.hadoop.hbase.favored.FavoredNodesPlan) ServerName(org.apache.hadoop.hbase.ServerName) HashMap(java.util.HashMap) Map(java.util.Map)

Example 7 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class AssignmentVerificationReport method fillUp.

public void fillUp(TableName tableName, SnapshotOfRegionAssignmentFromMeta snapshot, Map<String, Map<String, Float>> regionLocalityMap) {
    // Set the table name
    this.tableName = tableName;
    // Get all the regions for this table
    List<RegionInfo> regionInfoList = snapshot.getTableToRegionMap().get(tableName);
    // Get the total region num for the current table
    this.totalRegions = regionInfoList.size();
    // Get the existing assignment plan
    FavoredNodesPlan favoredNodesAssignment = snapshot.getExistingAssignmentPlan();
    // Get the region to region server mapping
    Map<RegionInfo, ServerName> currentAssignment = snapshot.getRegionToRegionServerMap();
    // Initialize the server to its hosing region counter map
    Map<ServerName, Integer> serverToHostingRegionCounterMap = new HashMap<>();
    Map<ServerName, Integer> primaryRSToRegionCounterMap = new HashMap<>();
    Map<ServerName, Set<ServerName>> primaryToSecTerRSMap = new HashMap<>();
    // Also keep tracker of the most loaded and least loaded region servers
    for (RegionInfo region : regionInfoList) {
        try {
            ServerName currentRS = currentAssignment.get(region);
            // Handle unassigned regions
            if (currentRS == null) {
                unAssignedRegionsList.add(region);
                continue;
            }
            // Keep updating the server to is hosting region counter map
            Integer hostRegionCounter = serverToHostingRegionCounterMap.get(currentRS);
            if (hostRegionCounter == null) {
                hostRegionCounter = Integer.valueOf(0);
            }
            hostRegionCounter = hostRegionCounter.intValue() + 1;
            serverToHostingRegionCounterMap.put(currentRS, hostRegionCounter);
            // Get the favored nodes from the assignment plan and verify it.
            List<ServerName> favoredNodes = favoredNodesAssignment.getFavoredNodes(region);
            if (favoredNodes == null || favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
                regionsWithoutValidFavoredNodes.add(region);
                continue;
            }
            // Get the primary, secondary and tertiary region server
            ServerName primaryRS = favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal());
            ServerName secondaryRS = favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal());
            ServerName tertiaryRS = favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal());
            // Update the primary rs to its region set map
            Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS);
            if (regionCounter == null) {
                regionCounter = Integer.valueOf(0);
            }
            regionCounter = regionCounter.intValue() + 1;
            primaryRSToRegionCounterMap.put(primaryRS, regionCounter);
            // Update the primary rs to secondary and tertiary rs map
            Set<ServerName> secAndTerSet = primaryToSecTerRSMap.get(primaryRS);
            if (secAndTerSet == null) {
                secAndTerSet = new HashSet<>();
            }
            secAndTerSet.add(secondaryRS);
            secAndTerSet.add(tertiaryRS);
            primaryToSecTerRSMap.put(primaryRS, secAndTerSet);
            // Get the position of the current region server in the favored nodes list
            FavoredNodesPlan.Position favoredNodePosition = FavoredNodesPlan.getFavoredServerPosition(favoredNodes, currentRS);
            // Handle the non favored assignment.
            if (favoredNodePosition == null) {
                nonFavoredAssignedRegionList.add(region);
                continue;
            }
            // Increase the favored nodes assignment.
            this.favoredNodes[favoredNodePosition.ordinal()]++;
            totalFavoredAssignments++;
            // Summary the locality information for each favored nodes
            if (regionLocalityMap != null) {
                // Set the enforce locality as true;
                this.enforceLocality = true;
                // Get the region degree locality map
                Map<String, Float> regionDegreeLocalityMap = regionLocalityMap.get(region.getEncodedName());
                if (regionDegreeLocalityMap == null) {
                    // ignore the region which doesn't have any store files.
                    continue;
                }
                // Get the locality summary for each favored nodes
                for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
                    ServerName favoredNode = favoredNodes.get(p.ordinal());
                    // Get the locality for the current favored nodes
                    Float locality = regionDegreeLocalityMap.get(favoredNode.getHostname());
                    if (locality != null) {
                        this.favoredNodesLocalitySummary[p.ordinal()] += locality;
                    }
                }
                // Get the locality summary for the current region server
                Float actualLocality = regionDegreeLocalityMap.get(currentRS.getHostname());
                if (actualLocality != null) {
                    this.actualLocalitySummary += actualLocality;
                }
            }
        } catch (Exception e) {
            LOG.error("Cannot verify the region assignment for region " + ((region == null) ? " null " : region.getRegionNameAsString()) + "because of " + e);
        }
    }
    float dispersionScoreSummary = 0;
    float dispersionNumSummary = 0;
    // Calculate the secondary score for each primary region server
    for (Map.Entry<ServerName, Integer> entry : primaryRSToRegionCounterMap.entrySet()) {
        ServerName primaryRS = entry.getKey();
        Integer regionsOnPrimary = entry.getValue();
        // Process the dispersion number and score
        float dispersionScore = 0;
        int dispersionNum = 0;
        if (primaryToSecTerRSMap.get(primaryRS) != null && regionsOnPrimary.intValue() != 0) {
            dispersionNum = primaryToSecTerRSMap.get(primaryRS).size();
            dispersionScore = dispersionNum / ((float) regionsOnPrimary.intValue() * 2);
        }
        // Update the max dispersion score
        if (dispersionScore > this.maxDispersionScore) {
            this.maxDispersionScoreServerSet.clear();
            this.maxDispersionScoreServerSet.add(primaryRS);
            this.maxDispersionScore = dispersionScore;
        } else if (dispersionScore == this.maxDispersionScore) {
            this.maxDispersionScoreServerSet.add(primaryRS);
        }
        // Update the max dispersion num
        if (dispersionNum > this.maxDispersionNum) {
            this.maxDispersionNumServerSet.clear();
            this.maxDispersionNumServerSet.add(primaryRS);
            this.maxDispersionNum = dispersionNum;
        } else if (dispersionNum == this.maxDispersionNum) {
            this.maxDispersionNumServerSet.add(primaryRS);
        }
        // Update the min dispersion score
        if (dispersionScore < this.minDispersionScore) {
            this.minDispersionScoreServerSet.clear();
            this.minDispersionScoreServerSet.add(primaryRS);
            this.minDispersionScore = dispersionScore;
        } else if (dispersionScore == this.minDispersionScore) {
            this.minDispersionScoreServerSet.add(primaryRS);
        }
        // Update the min dispersion num
        if (dispersionNum < this.minDispersionNum) {
            this.minDispersionNumServerSet.clear();
            this.minDispersionNumServerSet.add(primaryRS);
            this.minDispersionNum = dispersionNum;
        } else if (dispersionNum == this.minDispersionNum) {
            this.minDispersionNumServerSet.add(primaryRS);
        }
        dispersionScoreSummary += dispersionScore;
        dispersionNumSummary += dispersionNum;
    }
    // Update the avg dispersion score
    if (primaryRSToRegionCounterMap.keySet().size() != 0) {
        this.avgDispersionScore = dispersionScoreSummary / (float) primaryRSToRegionCounterMap.keySet().size();
        this.avgDispersionNum = dispersionNumSummary / (float) primaryRSToRegionCounterMap.keySet().size();
    }
    // Fill up the most loaded and least loaded region server information
    for (Map.Entry<ServerName, Integer> entry : serverToHostingRegionCounterMap.entrySet()) {
        ServerName currentRS = entry.getKey();
        int hostRegionCounter = entry.getValue().intValue();
        // Update the most loaded region server list and maxRegionsOnRS
        if (hostRegionCounter > this.maxRegionsOnRS) {
            maxRegionsOnRS = hostRegionCounter;
            this.mostLoadedRSSet.clear();
            this.mostLoadedRSSet.add(currentRS);
        } else if (hostRegionCounter == this.maxRegionsOnRS) {
            this.mostLoadedRSSet.add(currentRS);
        }
        // Update the least loaded region server list and minRegionsOnRS
        if (hostRegionCounter < this.minRegionsOnRS) {
            this.minRegionsOnRS = hostRegionCounter;
            this.leastLoadedRSSet.clear();
            this.leastLoadedRSSet.add(currentRS);
        } else if (hostRegionCounter == this.minRegionsOnRS) {
            this.leastLoadedRSSet.add(currentRS);
        }
    }
    // and total region servers
    this.totalRegionServers = serverToHostingRegionCounterMap.keySet().size();
    this.avgRegionsOnRS = (totalRegionServers == 0) ? 0 : (totalRegions / (float) totalRegionServers);
    // Set the isFilledUp as true
    isFilledUp = true;
}
Also used : Set(java.util.Set) HashSet(java.util.HashSet) HashMap(java.util.HashMap) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) FavoredNodesPlan(org.apache.hadoop.hbase.favored.FavoredNodesPlan) ServerName(org.apache.hadoop.hbase.ServerName) HashMap(java.util.HashMap) Map(java.util.Map)

Example 8 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class RegionPlacementMaintainer method main.

public static void main(String[] args) throws IOException {
    Options opt = new Options();
    opt.addOption("w", "write", false, "write the assignments to hbase:meta only");
    opt.addOption("u", "update", false, "update the assignments to hbase:meta and RegionServers together");
    opt.addOption("n", "dry-run", false, "do not write assignments to META");
    opt.addOption("v", "verify", false, "verify current assignments against META");
    opt.addOption("p", "print", false, "print the current assignment plan in META");
    opt.addOption("h", "help", false, "print usage");
    opt.addOption("d", "verification-details", false, "print the details of verification report");
    opt.addOption("zk", true, "to set the zookeeper quorum");
    opt.addOption("fs", true, "to set HDFS");
    opt.addOption("hbase_root", true, "to set hbase_root directory");
    opt.addOption("overwrite", false, "overwrite the favored nodes for a single region," + "for example: -update -r regionName -f server1:port,server2:port,server3:port");
    opt.addOption("r", true, "The region name that needs to be updated");
    opt.addOption("f", true, "The new favored nodes");
    opt.addOption("tables", true, "The list of table names splitted by ',' ;" + "For example: -tables: t1,t2,...,tn");
    opt.addOption("l", "locality", true, "enforce the maximum locality");
    opt.addOption("m", "min-move", true, "enforce minimum assignment move");
    opt.addOption("diff", false, "calculate difference between assignment plans");
    opt.addOption("munkres", false, "use munkres to place secondaries and tertiaries");
    opt.addOption("ld", "locality-dispersion", false, "print locality and dispersion " + "information for current plan");
    try {
        CommandLine cmd = new GnuParser().parse(opt, args);
        Configuration conf = HBaseConfiguration.create();
        boolean enforceMinAssignmentMove = true;
        boolean enforceLocality = true;
        boolean verificationDetails = false;
        // Read all the options
        if ((cmd.hasOption("l") && cmd.getOptionValue("l").equalsIgnoreCase("false")) || (cmd.hasOption("locality") && cmd.getOptionValue("locality").equalsIgnoreCase("false"))) {
            enforceLocality = false;
        }
        if ((cmd.hasOption("m") && cmd.getOptionValue("m").equalsIgnoreCase("false")) || (cmd.hasOption("min-move") && cmd.getOptionValue("min-move").equalsIgnoreCase("false"))) {
            enforceMinAssignmentMove = false;
        }
        if (cmd.hasOption("zk")) {
            conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue("zk"));
            LOG.info("Setting the zk quorum: " + conf.get(HConstants.ZOOKEEPER_QUORUM));
        }
        if (cmd.hasOption("fs")) {
            conf.set(FileSystem.FS_DEFAULT_NAME_KEY, cmd.getOptionValue("fs"));
            LOG.info("Setting the HDFS: " + conf.get(FileSystem.FS_DEFAULT_NAME_KEY));
        }
        if (cmd.hasOption("hbase_root")) {
            conf.set(HConstants.HBASE_DIR, cmd.getOptionValue("hbase_root"));
            LOG.info("Setting the hbase root directory: " + conf.get(HConstants.HBASE_DIR));
        }
        // Create the region placement obj
        try (RegionPlacementMaintainer rp = new RegionPlacementMaintainer(conf, enforceLocality, enforceMinAssignmentMove)) {
            if (cmd.hasOption("d") || cmd.hasOption("verification-details")) {
                verificationDetails = true;
            }
            if (cmd.hasOption("tables")) {
                String tableNameListStr = cmd.getOptionValue("tables");
                String[] tableNames = StringUtils.split(tableNameListStr, ",");
                rp.setTargetTableName(tableNames);
            }
            if (cmd.hasOption("munkres")) {
                USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY = true;
            }
            // Read all the modes
            if (cmd.hasOption("v") || cmd.hasOption("verify")) {
                // Verify the region placement.
                rp.verifyRegionPlacement(verificationDetails);
            } else if (cmd.hasOption("n") || cmd.hasOption("dry-run")) {
                // Generate the assignment plan only without updating the hbase:meta and RS
                FavoredNodesPlan plan = rp.getNewAssignmentPlan();
                printAssignmentPlan(plan);
            } else if (cmd.hasOption("w") || cmd.hasOption("write")) {
                // Generate the new assignment plan
                FavoredNodesPlan plan = rp.getNewAssignmentPlan();
                // Print the new assignment plan
                printAssignmentPlan(plan);
                // Write the new assignment plan to META
                rp.updateAssignmentPlanToMeta(plan);
            } else if (cmd.hasOption("u") || cmd.hasOption("update")) {
                // Generate the new assignment plan
                FavoredNodesPlan plan = rp.getNewAssignmentPlan();
                // Print the new assignment plan
                printAssignmentPlan(plan);
                // Update the assignment to hbase:meta and Region Servers
                rp.updateAssignmentPlan(plan);
            } else if (cmd.hasOption("diff")) {
                FavoredNodesPlan newPlan = rp.getNewAssignmentPlan();
                Map<String, Map<String, Float>> locality = FSUtils.getRegionDegreeLocalityMappingFromFS(conf);
                Map<TableName, Integer> movesPerTable = rp.getRegionsMovement(newPlan);
                rp.checkDifferencesWithOldPlan(movesPerTable, locality, newPlan);
                System.out.println("Do you want to update the assignment plan? [y/n]");
                Scanner s = new Scanner(System.in);
                String input = s.nextLine().trim();
                if (input.equals("y")) {
                    System.out.println("Updating assignment plan...");
                    rp.updateAssignmentPlan(newPlan);
                }
                s.close();
            } else if (cmd.hasOption("ld")) {
                Map<String, Map<String, Float>> locality = FSUtils.getRegionDegreeLocalityMappingFromFS(conf);
                rp.printLocalityAndDispersionForCurrentPlan(locality);
            } else if (cmd.hasOption("p") || cmd.hasOption("print")) {
                FavoredNodesPlan plan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
                printAssignmentPlan(plan);
            } else if (cmd.hasOption("overwrite")) {
                if (!cmd.hasOption("f") || !cmd.hasOption("r")) {
                    throw new IllegalArgumentException("Please specify: " + " -update -r regionName -f server1:port,server2:port,server3:port");
                }
                String regionName = cmd.getOptionValue("r");
                String favoredNodesStr = cmd.getOptionValue("f");
                LOG.info("Going to update the region " + regionName + " with the new favored nodes " + favoredNodesStr);
                List<ServerName> favoredNodes = null;
                RegionInfo regionInfo = rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap().get(regionName);
                if (regionInfo == null) {
                    LOG.error("Cannot find the region " + regionName + " from the META");
                } else {
                    try {
                        favoredNodes = getFavoredNodeList(favoredNodesStr);
                    } catch (IllegalArgumentException e) {
                        LOG.error("Cannot parse the invalid favored nodes because " + e);
                    }
                    FavoredNodesPlan newPlan = new FavoredNodesPlan();
                    newPlan.updateFavoredNodesMap(regionInfo, favoredNodes);
                    rp.updateAssignmentPlan(newPlan);
                }
            } else {
                printHelp(opt);
            }
        }
    } catch (ParseException e) {
        printHelp(opt);
    }
}
Also used : Options(org.apache.hbase.thirdparty.org.apache.commons.cli.Options) Scanner(java.util.Scanner) Configuration(org.apache.hadoop.conf.Configuration) HBaseConfiguration(org.apache.hadoop.hbase.HBaseConfiguration) GnuParser(org.apache.hbase.thirdparty.org.apache.commons.cli.GnuParser) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) FavoredNodesPlan(org.apache.hadoop.hbase.favored.FavoredNodesPlan) TableName(org.apache.hadoop.hbase.TableName) CommandLine(org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine) ArrayList(java.util.ArrayList) List(java.util.List) ParseException(org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException) HashMap(java.util.HashMap) Map(java.util.Map) TreeMap(java.util.TreeMap)

Example 9 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class RegionPlacementMaintainer method printLocalityAndDispersionForCurrentPlan.

public void printLocalityAndDispersionForCurrentPlan(Map<String, Map<String, Float>> regionLocalityMap) throws IOException {
    SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
    FavoredNodesPlan assignmentPlan = snapshot.getExistingAssignmentPlan();
    Set<TableName> tables = snapshot.getTableSet();
    Map<TableName, List<RegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap();
    for (TableName table : tables) {
        float[] locality = new float[3];
        if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
            continue;
        }
        List<RegionInfo> regions = tableToRegionsMap.get(table);
        for (RegionInfo currentRegion : regions) {
            Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName());
            if (regionLocality == null) {
                continue;
            }
            List<ServerName> servers = assignmentPlan.getFavoredNodes(currentRegion);
            if (servers != null) {
                int i = 0;
                for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
                    ServerName server = servers.get(p.ordinal());
                    Float currentLocality = 0f;
                    if (servers != null) {
                        currentLocality = regionLocality.get(server.getHostname());
                        if (currentLocality == null) {
                            currentLocality = 0f;
                        }
                        locality[i] += currentLocality;
                    }
                    i++;
                }
            }
        }
        for (int i = 0; i < locality.length; i++) {
            String copy = null;
            if (i == 0) {
                copy = "primary";
            } else if (i == 1) {
                copy = "secondary";
            } else if (i == 2) {
                copy = "tertiary";
            }
            float avgLocality = 100 * locality[i] / regions.size();
            LOG.info("For Table: " + table + " ; #Total Regions: " + regions.size() + " ; The average locality for " + copy + " is " + avgLocality + " %");
        }
        printDispersionScores(table, snapshot, regions.size(), null, false);
    }
}
Also used : RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) FavoredNodesPlan(org.apache.hadoop.hbase.favored.FavoredNodesPlan) TableName(org.apache.hadoop.hbase.TableName) ServerName(org.apache.hadoop.hbase.ServerName) ArrayList(java.util.ArrayList) List(java.util.List)

Example 10 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class RegionPlacementMaintainer method checkDifferencesWithOldPlan.

/**
 * Compares two plans and check whether the locality dropped or increased
 * (prints the information as a string) also prints the baseline locality
 *
 * @param movesPerTable - how many primary regions will move per table
 * @param regionLocalityMap - locality map from FS
 * @param newPlan - new assignment plan
 * @throws IOException
 */
public void checkDifferencesWithOldPlan(Map<TableName, Integer> movesPerTable, Map<String, Map<String, Float>> regionLocalityMap, FavoredNodesPlan newPlan) throws IOException {
    // localities for primary, secondary and tertiary
    SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
    FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan();
    Set<TableName> tables = snapshot.getTableSet();
    Map<TableName, List<RegionInfo>> tableToRegionsMap = snapshot.getTableToRegionMap();
    for (TableName table : tables) {
        float[] deltaLocality = new float[3];
        float[] locality = new float[3];
        if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
            continue;
        }
        List<RegionInfo> regions = tableToRegionsMap.get(table);
        System.out.println("==================================================");
        System.out.println("Assignment Plan Projection Report For Table: " + table);
        System.out.println("\t Total regions: " + regions.size());
        System.out.println("\t" + movesPerTable.get(table) + " primaries will move due to their primary has changed");
        for (RegionInfo currentRegion : regions) {
            Map<String, Float> regionLocality = regionLocalityMap.get(currentRegion.getEncodedName());
            if (regionLocality == null) {
                continue;
            }
            List<ServerName> oldServers = oldPlan.getFavoredNodes(currentRegion);
            List<ServerName> newServers = newPlan.getFavoredNodes(currentRegion);
            if (newServers != null && oldServers != null) {
                int i = 0;
                for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) {
                    ServerName newServer = newServers.get(p.ordinal());
                    ServerName oldServer = oldServers.get(p.ordinal());
                    Float oldLocality = 0f;
                    if (oldServers != null) {
                        oldLocality = regionLocality.get(oldServer.getHostname());
                        if (oldLocality == null) {
                            oldLocality = 0f;
                        }
                        locality[i] += oldLocality;
                    }
                    Float newLocality = regionLocality.get(newServer.getHostname());
                    if (newLocality == null) {
                        newLocality = 0f;
                    }
                    deltaLocality[i] += newLocality - oldLocality;
                    i++;
                }
            }
        }
        DecimalFormat df = new java.text.DecimalFormat("#.##");
        for (int i = 0; i < deltaLocality.length; i++) {
            System.out.print("\t\t Baseline locality for ");
            if (i == 0) {
                System.out.print("primary ");
            } else if (i == 1) {
                System.out.print("secondary ");
            } else if (i == 2) {
                System.out.print("tertiary ");
            }
            System.out.println(df.format(100 * locality[i] / regions.size()) + "%");
            System.out.print("\t\t Locality will change with the new plan: ");
            System.out.println(df.format(100 * deltaLocality[i] / regions.size()) + "%");
        }
        System.out.println("\t Baseline dispersion");
        printDispersionScores(table, snapshot, regions.size(), null, true);
        System.out.println("\t Projected dispersion");
        printDispersionScores(table, snapshot, regions.size(), newPlan, true);
    }
}
Also used : DecimalFormat(java.text.DecimalFormat) RegionInfo(org.apache.hadoop.hbase.client.RegionInfo) FavoredNodesPlan(org.apache.hadoop.hbase.favored.FavoredNodesPlan) TableName(org.apache.hadoop.hbase.TableName) ServerName(org.apache.hadoop.hbase.ServerName) ArrayList(java.util.ArrayList) List(java.util.List)

Aggregations

FavoredNodesPlan (org.apache.hadoop.hbase.favored.FavoredNodesPlan)11 RegionInfo (org.apache.hadoop.hbase.client.RegionInfo)8 ArrayList (java.util.ArrayList)7 HashMap (java.util.HashMap)7 List (java.util.List)7 ServerName (org.apache.hadoop.hbase.ServerName)7 Map (java.util.Map)6 TableName (org.apache.hadoop.hbase.TableName)6 TreeMap (java.util.TreeMap)3 ParseException (org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException)3 IOException (java.io.IOException)2 HashSet (java.util.HashSet)2 Set (java.util.Set)2 DecimalFormat (java.text.DecimalFormat)1 Scanner (java.util.Scanner)1 Configuration (org.apache.hadoop.conf.Configuration)1 HBaseConfiguration (org.apache.hadoop.hbase.HBaseConfiguration)1 AsyncRegionServerAdmin (org.apache.hadoop.hbase.client.AsyncRegionServerAdmin)1 UpdateFavoredNodesRequest (org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest)1 UpdateFavoredNodesResponse (org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse)1