Examples with RegionInfo org.apache.hadoop.hbase.client.RegionInfo used on opensource projects

Example 36 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class FavoredStochasticBalancer method segregateRegionsAndAssignRegionsWithFavoredNodes.

/**
 * Return a pair - one with assignments when favored nodes are present and another with regions
 * without favored nodes.
 */
private Pair<Map<ServerName, List<RegionInfo>>, List<RegionInfo>> segregateRegionsAndAssignRegionsWithFavoredNodes(Collection<RegionInfo> regions, List<ServerName> onlineServers) throws HBaseIOException {
    // Since we expect FN to be present most of the time, lets create map with same size
    Map<ServerName, List<RegionInfo>> assignmentMapForFavoredNodes = new HashMap<>(onlineServers.size());
    List<RegionInfo> regionsWithNoFavoredNodes = new ArrayList<>();
    for (RegionInfo region : regions) {
        List<ServerName> favoredNodes = fnm.getFavoredNodes(region);
        ServerName primaryHost = null;
        ServerName secondaryHost = null;
        ServerName tertiaryHost = null;
        if (favoredNodes != null && !favoredNodes.isEmpty()) {
            for (ServerName s : favoredNodes) {
                ServerName serverWithLegitStartCode = getServerFromFavoredNode(onlineServers, s);
                if (serverWithLegitStartCode != null) {
                    FavoredNodesPlan.Position position = FavoredNodesPlan.getFavoredServerPosition(favoredNodes, s);
                    if (Position.PRIMARY.equals(position)) {
                        primaryHost = serverWithLegitStartCode;
                    } else if (Position.SECONDARY.equals(position)) {
                        secondaryHost = serverWithLegitStartCode;
                    } else if (Position.TERTIARY.equals(position)) {
                        tertiaryHost = serverWithLegitStartCode;
                    }
                }
            }
            assignRegionToAvailableFavoredNode(assignmentMapForFavoredNodes, region, primaryHost, secondaryHost, tertiaryHost);
        } else {
            regionsWithNoFavoredNodes.add(region);
        }
    }
    return new Pair<>(assignmentMapForFavoredNodes, regionsWithNoFavoredNodes);
}

Example 37 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class FavoredStochasticBalancer method roundRobinAssignment.

/**
 * Round robin assignment: Segregate the regions into two types:
 *
 * 1. The regions that have favored node assignment where at least one of the favored node
 * is still alive. In this case, try to adhere to the current favored nodes assignment as
 * much as possible - i.e., if the current primary is gone, then make the secondary or
 * tertiary as the new host for the region (based on their current load). Note that we don't
 * change the favored node assignments here (even though one or more favored node is
 * currently down). That will be done by the admin operations.
 *
 * 2. The regions that currently don't have favored node assignments. Generate favored nodes
 * for them and then assign. Generate the primary fn in round robin fashion and generate
 * secondary and tertiary as per favored nodes constraints.
 */
@Override
@NonNull
public Map<ServerName, List<RegionInfo>> roundRobinAssignment(List<RegionInfo> regions, List<ServerName> servers) throws HBaseIOException {
    metricsBalancer.incrMiscInvocations();
    Map<ServerName, List<RegionInfo>> assignmentMap = new HashMap<>();
    if (regions.isEmpty()) {
        return assignmentMap;
    }
    Set<RegionInfo> regionSet = new HashSet<>(regions);
    try {
        FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, rackManager);
        helper.initialize();
        Set<RegionInfo> systemRegions = FavoredNodesManager.filterNonFNApplicableRegions(regionSet);
        regionSet.removeAll(systemRegions);
        // Assign all system regions
        Map<ServerName, List<RegionInfo>> systemAssignments = super.roundRobinAssignment(Lists.newArrayList(systemRegions), servers);
        // Segregate favored and non-favored nodes regions and assign accordingly.
        Pair<Map<ServerName, List<RegionInfo>>, List<RegionInfo>> segregatedRegions = segregateRegionsAndAssignRegionsWithFavoredNodes(regionSet, servers);
        Map<ServerName, List<RegionInfo>> regionsWithFavoredNodesMap = segregatedRegions.getFirst();
        Map<ServerName, List<RegionInfo>> regionsWithoutFN = generateFNForRegionsWithoutFN(helper, segregatedRegions.getSecond());
        // merge the assignment maps
        mergeAssignmentMaps(assignmentMap, systemAssignments);
        mergeAssignmentMaps(assignmentMap, regionsWithFavoredNodesMap);
        mergeAssignmentMaps(assignmentMap, regionsWithoutFN);
    } catch (Exception ex) {
        throw new HBaseIOException("Encountered exception while doing favored-nodes assignment " + ex + " Falling back to regular assignment", ex);
    }
    return assignmentMap;
}

Example 38 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class FavoredStochasticBalancer method mergeAssignmentMaps.

private void mergeAssignmentMaps(Map<ServerName, List<RegionInfo>> assignmentMap, Map<ServerName, List<RegionInfo>> otherAssignments) {
    if (otherAssignments == null || otherAssignments.isEmpty()) {
        return;
    }
    for (Map.Entry<ServerName, List<RegionInfo>> entry : otherAssignments.entrySet()) {
        ServerName sn = entry.getKey();
        List<RegionInfo> regionsList = entry.getValue();
        if (assignmentMap.get(sn) == null) {
            assignmentMap.put(sn, Lists.newArrayList(regionsList));
        } else {
            assignmentMap.get(sn).addAll(regionsList);
        }
    }
}

Example 39 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class BaseLoadBalancer method retainAssignment.

/**
 * Generates a bulk assignment startup plan, attempting to reuse the existing
 * assignment information from META, but adjusting for the specified list of
 * available/online servers available for assignment.
 * <p>
 * Takes a map of all regions to their existing assignment from META. Also
 * takes a list of online servers for regions to be assigned to. Attempts to
 * retain all assignment, so in some instances initial assignment will not be
 * completely balanced.
 * <p>
 * Any leftover regions without an existing server to be assigned to will be
 * assigned randomly to available servers.
 *
 * @param regions regions and existing assignment from meta
 * @param servers available servers
 * @return map of servers and regions to be assigned to them, or emptyMap if no
 *           assignment is possible (ie. no servers)
 */
@Override
@NonNull
public Map<ServerName, List<RegionInfo>> retainAssignment(Map<RegionInfo, ServerName> regions, List<ServerName> servers) throws HBaseIOException {
    // Update metrics
    metricsBalancer.incrMiscInvocations();
    int numServers = servers == null ? 0 : servers.size();
    if (numServers == 0) {
        LOG.warn("Wanted to do retain assignment but no servers to assign to");
        return Collections.singletonMap(BOGUS_SERVER_NAME, new ArrayList<>(regions.keySet()));
    }
    if (numServers == 1) {
        // Only one server, nothing fancy we can do here
        return Collections.singletonMap(servers.get(0), new ArrayList<>(regions.keySet()));
    }
    // Group all of the old assignments by their hostname.
    // We can't group directly by ServerName since the servers all have
    // new start-codes.
    // Group the servers by their hostname. It's possible we have multiple
    // servers on the same host on different ports.
    Map<ServerName, List<RegionInfo>> assignments = new HashMap<>();
    ArrayListMultimap<String, ServerName> serversByHostname = ArrayListMultimap.create();
    for (ServerName server : servers) {
        assignments.put(server, new ArrayList<>());
        serversByHostname.put(server.getHostnameLowerCase(), server);
    }
    // Collection of the hostnames that used to have regions
    // assigned, but for which we no longer have any RS running
    // after the cluster restart.
    Set<String> oldHostsNoLongerPresent = Sets.newTreeSet();
    // If the old servers aren't present, lets assign those regions later.
    List<RegionInfo> randomAssignRegions = Lists.newArrayList();
    int numRandomAssignments = 0;
    int numRetainedAssigments = 0;
    for (Map.Entry<RegionInfo, ServerName> entry : regions.entrySet()) {
        RegionInfo region = entry.getKey();
        ServerName oldServerName = entry.getValue();
        List<ServerName> localServers = new ArrayList<>();
        if (oldServerName != null) {
            localServers = serversByHostname.get(oldServerName.getHostnameLowerCase());
        }
        if (localServers.isEmpty()) {
            // No servers on the new cluster match up with this hostname, assign randomly, later.
            randomAssignRegions.add(region);
            if (oldServerName != null) {
                oldHostsNoLongerPresent.add(oldServerName.getHostnameLowerCase());
            }
        } else if (localServers.size() == 1) {
            // the usual case - one new server on same host
            ServerName target = localServers.get(0);
            assignments.get(target).add(region);
            numRetainedAssigments++;
        } else {
            // multiple new servers in the cluster on this same host
            if (localServers.contains(oldServerName)) {
                assignments.get(oldServerName).add(region);
                numRetainedAssigments++;
            } else {
                ServerName target = null;
                for (ServerName tmp : localServers) {
                    if (tmp.getPort() == oldServerName.getPort()) {
                        target = tmp;
                        assignments.get(tmp).add(region);
                        numRetainedAssigments++;
                        break;
                    }
                }
                if (target == null) {
                    randomAssignRegions.add(region);
                }
            }
        }
    }
    // If servers from prior assignment aren't present, then lets do randomAssignment on regions.
    if (randomAssignRegions.size() > 0) {
        BalancerClusterState cluster = createCluster(servers, regions.keySet());
        for (Map.Entry<ServerName, List<RegionInfo>> entry : assignments.entrySet()) {
            ServerName sn = entry.getKey();
            for (RegionInfo region : entry.getValue()) {
                cluster.doAssignRegion(region, sn);
            }
        }
        for (RegionInfo region : randomAssignRegions) {
            ServerName target = randomAssignment(cluster, region, servers);
            assignments.get(target).add(region);
            numRandomAssignments++;
        }
    }
    String randomAssignMsg = "";
    if (numRandomAssignments > 0) {
        randomAssignMsg = numRandomAssignments + " regions were assigned " + "to random hosts, since the old hosts for these regions are no " + "longer present in the cluster. These hosts were:\n  " + Joiner.on("\n  ").join(oldHostsNoLongerPresent);
    }
    LOG.info("Reassigned " + regions.size() + " regions. " + numRetainedAssigments + " retained the pre-restart assignment. " + randomAssignMsg);
    return Collections.unmodifiableMap(assignments);
}

Example 40 with RegionInfo

use of org.apache.hadoop.hbase.client.RegionInfo in project hbase by apache.

the class RegionHDFSBlockLocationFinder method scheduleFullRefresh.

/**
 * Refresh all the region locations.
 * @return true if user created regions got refreshed.
 */
private boolean scheduleFullRefresh() {
    ClusterInfoProvider service = this.provider;
    // Protect from anything being null while starting up.
    if (service == null) {
        return false;
    }
    // TODO: Should this refresh all the regions or only the ones assigned?
    boolean includesUserTables = false;
    for (final RegionInfo hri : service.getAssignedRegions()) {
        cache.refresh(hri);
        includesUserTables |= !hri.getTable().isSystemTable();
    }
    return includesUserTables;
}