Examples with FavoredNodesPlan org.apache.hadoop.hbase.favored.FavoredNodesPlan used on opensource projects

Example 1 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class RegionPlacementMaintainer method updateAssignmentPlanToRegionServers.

/**
 * Update the assignment plan to all the region servers
 * @param plan
 * @throws IOException
 */
private void updateAssignmentPlanToRegionServers(FavoredNodesPlan plan) throws IOException {
    LOG.info("Start to update the region servers with the new assignment plan");
    // Get the region to region server map
    Map<ServerName, List<RegionInfo>> currentAssignment = this.getRegionAssignmentSnapshot().getRegionServerToRegionMap();
    // track of the failed and succeeded updates
    int succeededNum = 0;
    Map<ServerName, Exception> failedUpdateMap = new HashMap<>();
    for (Map.Entry<ServerName, List<RegionInfo>> entry : currentAssignment.entrySet()) {
        List<Pair<RegionInfo, List<ServerName>>> regionUpdateInfos = new ArrayList<>();
        try {
            // Keep track of the favored updates for the current region server
            FavoredNodesPlan singleServerPlan = null;
            // Find out all the updates for the current region server
            for (RegionInfo region : entry.getValue()) {
                List<ServerName> favoredServerList = plan.getFavoredNodes(region);
                if (favoredServerList != null && favoredServerList.size() == FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
                    // Create the single server plan if necessary
                    if (singleServerPlan == null) {
                        singleServerPlan = new FavoredNodesPlan();
                    }
                    // Update the single server update
                    singleServerPlan.updateFavoredNodesMap(region, favoredServerList);
                    regionUpdateInfos.add(new Pair<>(region, favoredServerList));
                }
            }
            if (singleServerPlan != null) {
                // Update the current region server with its updated favored nodes
                AsyncRegionServerAdmin rsAdmin = getConnection().getRegionServerAdmin(entry.getKey());
                UpdateFavoredNodesRequest request = RequestConverter.buildUpdateFavoredNodesRequest(regionUpdateInfos);
                UpdateFavoredNodesResponse updateFavoredNodesResponse = FutureUtils.get(rsAdmin.updateFavoredNodes(request));
                LOG.info("Region server " + FutureUtils.get(rsAdmin.getServerInfo(RequestConverter.buildGetServerInfoRequest())).getServerInfo() + " has updated " + updateFavoredNodesResponse.getResponse() + " / " + singleServerPlan.size() + " regions with the assignment plan");
                succeededNum++;
            }
        } catch (Exception e) {
            failedUpdateMap.put(entry.getKey(), e);
        }
    }
    // log the succeeded updates
    LOG.info("Updated " + succeededNum + " region servers with " + "the new assignment plan");
    // log the failed updates
    int failedNum = failedUpdateMap.size();
    if (failedNum != 0) {
        LOG.error("Failed to update the following + " + failedNum + " region servers with its corresponding favored nodes");
        for (Map.Entry<ServerName, Exception> entry : failedUpdateMap.entrySet()) {
            LOG.error("Failed to update " + entry.getKey().getAddress() + " because of " + entry.getValue().getMessage());
        }
    }
}

Example 2 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class RegionPlacementMaintainer method getRegionsMovement.

/**
 * Return how many regions will move per table since their primary RS will
 * change
 *
 * @param newPlan - new AssignmentPlan
 * @return how many primaries will move per table
 */
public Map<TableName, Integer> getRegionsMovement(FavoredNodesPlan newPlan) throws IOException {
    Map<TableName, Integer> movesPerTable = new HashMap<>();
    SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot();
    Map<TableName, List<RegionInfo>> tableToRegions = snapshot.getTableToRegionMap();
    FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan();
    Set<TableName> tables = snapshot.getTableSet();
    for (TableName table : tables) {
        int movedPrimaries = 0;
        if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
            continue;
        }
        List<RegionInfo> regions = tableToRegions.get(table);
        for (RegionInfo region : regions) {
            List<ServerName> oldServers = oldPlan.getFavoredNodes(region);
            List<ServerName> newServers = newPlan.getFavoredNodes(region);
            if (oldServers != null && newServers != null) {
                ServerName oldPrimary = oldServers.get(0);
                ServerName newPrimary = newServers.get(0);
                if (oldPrimary.compareTo(newPrimary) != 0) {
                    movedPrimaries++;
                }
            }
        }
        movesPerTable.put(table, movedPrimaries);
    }
    return movesPerTable;
}

Example 3 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class RegionPlacementMaintainer method getNewAssignmentPlan.

public FavoredNodesPlan getNewAssignmentPlan() throws IOException {
    // Get the current region assignment snapshot by scanning from the META
    SnapshotOfRegionAssignmentFromMeta assignmentSnapshot = this.getRegionAssignmentSnapshot();
    // Get the region locality map
    Map<String, Map<String, Float>> regionLocalityMap = null;
    if (this.enforceLocality) {
        regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf);
    }
    // Initialize the assignment plan
    FavoredNodesPlan plan = new FavoredNodesPlan();
    // Get the table to region mapping
    Map<TableName, List<RegionInfo>> tableToRegionMap = assignmentSnapshot.getTableToRegionMap();
    LOG.info("Start to generate the new assignment plan for the " + +tableToRegionMap.keySet().size() + " tables");
    for (TableName table : tableToRegionMap.keySet()) {
        try {
            if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) {
                continue;
            }
            // TODO: maybe run the placement in parallel for each table
            genAssignmentPlan(table, assignmentSnapshot, regionLocalityMap, plan, USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY);
        } catch (Exception e) {
            LOG.error("Get some exceptions for placing primary region server" + "for table " + table + " because " + e);
        }
    }
    LOG.info("Finish to generate the new assignment plan for the " + +tableToRegionMap.keySet().size() + " tables");
    return plan;
}

Example 4 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class TestRegionPlacement method testRegionPlacement.

@Ignore("Test for unfinished feature")
@Test
public void testRegionPlacement() throws Exception {
    String tableStr = "testRegionAssignment";
    TableName table = TableName.valueOf(tableStr);
    // Create a table with REGION_NUM regions.
    createTable(table, REGION_NUM);
    TEST_UTIL.waitTableAvailable(table);
    // Verify all the user regions are assigned to the primary region server
    // based on the plan
    verifyRegionOnPrimaryRS(REGION_NUM);
    FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
    // Verify all the region server are update with the latest favored nodes
    verifyRegionServerUpdated(currentPlan);
    // Test Case 2: To verify whether the region placement tools can
    // correctly update the new assignment plan to hbase:meta and Region Server.
    // The new assignment plan is generated by shuffle the existing assignment
    // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
    // Shuffle the plan by switching the secondary region server with
    // the tertiary.
    // Shuffle the secondary with tertiary favored nodes
    FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan, FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
    // Let the region placement update the hbase:meta and Region Servers
    rp.updateAssignmentPlan(shuffledPlan);
    // Verify the region assignment. There are supposed to no region reassignment
    // All the regions are still on the primary region server
    verifyRegionAssignment(shuffledPlan, 0, REGION_NUM);
    // Shuffle the plan by switching the primary with secondary and
    // verify the region reassignment is consistent with the plan.
    shuffledPlan = this.shuffleAssignmentPlan(currentPlan, FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
    // Let the region placement update the hbase:meta and Region Servers
    rp.updateAssignmentPlan(shuffledPlan);
    verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
    // also verify that the AssignmentVerificationReport has the correct information
    RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
    // we are interested in only one table (and hence one report)
    rp.setTargetTableName(new String[] { tableStr });
    List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
    AssignmentVerificationReport report = reports.get(0);
    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
    assertTrue(report.getUnassignedRegions().isEmpty());
    // Check when a RS stops, the regions get assigned to their secondary/tertiary
    killRandomServerAndVerifyAssignment();
    // also verify that the AssignmentVerificationReport has the correct information
    reports = rp.verifyRegionPlacement(false);
    report = reports.get(0);
    assertTrue(report.getRegionsWithoutValidFavoredNodes().isEmpty());
    assertTrue(report.getNonFavoredAssignedRegions().isEmpty());
    assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
    assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
    assertTrue("secondary " + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary " + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY), (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0 || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
    assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
    RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
}

Example 5 with FavoredNodesPlan

use of org.apache.hadoop.hbase.favored.FavoredNodesPlan in project hbase by apache.

the class TestRegionPlacement method shuffleAssignmentPlan.

/**
 * Shuffle the assignment plan by switching two favored node positions.
 * @param plan The assignment plan
 * @param p1 The first switch position
 * @param p2 The second switch position
 * @return the shuffled assignment plan
 */
private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan, FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) throws IOException {
    FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
    Map<String, RegionInfo> regionToHRegion = rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap();
    for (Map.Entry<String, List<ServerName>> entry : plan.getAssignmentMap().entrySet()) {
        // copy the server list from the original plan
        List<ServerName> shuffledServerList = new ArrayList<>();
        shuffledServerList.addAll(entry.getValue());
        // start to shuffle
        shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
        shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
        // update the plan
        shuffledPlan.updateFavoredNodesMap(regionToHRegion.get(entry.getKey()), shuffledServerList);
    }
    return shuffledPlan;
}