Examples with Address org.apache.hadoop.hbase.net.Address used on opensource projects

Example 6 with Address

use of org.apache.hadoop.hbase.net.Address in project hbase by apache.

the class RSGroupAdminServer method unassignRegionFromServers.

/**
   * @param servers the servers that will move to new group
   * @param targetGroupName the target group name
   * @param tables The regions of tables assigned to these servers will not unassign
   * @throws IOException
   */
private void unassignRegionFromServers(Set<Address> servers, String targetGroupName, Set<TableName> tables) throws IOException {
    boolean foundRegionsToUnassign;
    RSGroupInfo targetGrp = getRSGroupInfo(targetGroupName);
    Set<Address> allSevers = new HashSet<>(servers);
    do {
        foundRegionsToUnassign = false;
        for (Iterator<Address> iter = allSevers.iterator(); iter.hasNext(); ) {
            Address rs = iter.next();
            // Get regions that are associated with this server and filter regions by tables.
            List<HRegionInfo> regions = new ArrayList<>();
            for (HRegionInfo region : getRegions(rs)) {
                if (!tables.contains(region.getTable())) {
                    regions.add(region);
                }
            }
            LOG.info("Unassigning " + regions.size() + " region(s) from " + rs + " for server move to " + targetGroupName);
            if (!regions.isEmpty()) {
                for (HRegionInfo region : regions) {
                    // Regions might get assigned from tables of target group so we need to filter
                    if (!targetGrp.containsTable(region.getTable())) {
                        this.master.getAssignmentManager().unassign(region);
                        if (master.getAssignmentManager().getRegionStates().getRegionState(region).isFailedOpen()) {
                            continue;
                        }
                        foundRegionsToUnassign = true;
                    }
                }
            }
            if (!foundRegionsToUnassign) {
                iter.remove();
            }
        }
        try {
            rsGroupInfoManager.wait(1000);
        } catch (InterruptedException e) {
            LOG.warn("Sleep interrupted", e);
            Thread.currentThread().interrupt();
        }
    } while (foundRegionsToUnassign);
}

Example 7 with Address

use of org.apache.hadoop.hbase.net.Address in project hbase by apache.

the class TestRSGroupsBase method testMoveServersAndTables.

@Test
public void testMoveServersAndTables() throws Exception {
    final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()), 1);
    //create table
    final byte[] familyNameBytes = Bytes.toBytes("f");
    TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            List<String> regions = getTableRegionMap().get(tableName);
            if (regions == null)
                return false;
            return getTableRegionMap().get(tableName).size() >= 5;
        }
    });
    //get server which is not a member of new group
    ServerName targetServer = null;
    for (ServerName server : admin.getClusterStatus().getServers()) {
        if (!newGroup.containsServer(server.getAddress()) && !rsGroupAdmin.getRSGroupInfo("master").containsServer(server.getAddress())) {
            targetServer = server;
            break;
        }
    }
    //test fail bogus server move
    try {
        rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:9999")), Sets.newHashSet(tableName), newGroup.getName());
        fail("Bogus servers shouldn't have been successfully moved.");
    } catch (IOException ex) {
        String exp = "Source RSGroup for server foo:9999 does not exist.";
        String msg = "Expected '" + exp + "' in exception message: ";
        assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
    }
    //test fail server move
    try {
        rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP);
        fail("servers shouldn't have been successfully moved.");
    } catch (IOException ex) {
        String exp = "Target RSGroup " + RSGroupInfo.DEFAULT_GROUP + " is same as source " + RSGroupInfo.DEFAULT_GROUP + " RSGroup.";
        String msg = "Expected '" + exp + "' in exception message: ";
        assertTrue(msg + " " + ex.getMessage(), ex.getMessage().contains(exp));
    }
    //verify default group info
    Assert.assertEquals(3, rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
    Assert.assertEquals(4, rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
    //verify new group info
    Assert.assertEquals(1, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers().size());
    Assert.assertEquals(0, rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables().size());
    //get all region to move targetServer
    List<String> regionList = getTableRegionMap().get(tableName);
    for (String region : regionList) {
        // Lets move this region to the targetServer
        TEST_UTIL.getAdmin().move(Bytes.toBytes(HRegionInfo.encodeRegionName(Bytes.toBytes(region))), Bytes.toBytes(targetServer.getServerName()));
    }
    TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return getTableRegionMap().get(tableName) != null && getTableRegionMap().get(tableName).size() == 5 && getTableServerRegionMap().get(tableName).size() == 1 && admin.getClusterStatus().getRegionsInTransition().size() < 1;
        }
    });
    //verify that all region move to targetServer
    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
    //move targetServer and table to newGroup
    LOG.info("moving server and table to newGroup");
    rsGroupAdmin.moveServersAndTables(Sets.newHashSet(targetServer.getAddress()), Sets.newHashSet(tableName), newGroup.getName());
    //verify group change
    Assert.assertEquals(newGroup.getName(), rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName());
    //verify servers' not exist in old group
    Set<Address> defaultServers = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers();
    assertFalse(defaultServers.contains(targetServer.getAddress()));
    //verify servers' exist in new group
    Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers();
    assertTrue(newGroupServers.contains(targetServer.getAddress()));
    //verify tables' not exist in old group
    Set<TableName> defaultTables = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables();
    assertFalse(defaultTables.contains(tableName));
    //verify tables' exist in new group
    Set<TableName> newGroupTables = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getTables();
    assertTrue(newGroupTables.contains(tableName));
    //verify that all region still assgin on targetServer
    Assert.assertEquals(5, getTableServerRegionMap().get(tableName).get(targetServer).size());
}

Example 8 with Address

use of org.apache.hadoop.hbase.net.Address in project hbase by apache.

the class RSGroupAdminServer method moveServers.

@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE", justification = "Ignoring complaint because don't know what it is complaining about")
@Override
public void moveServers(Set<Address> servers, String targetGroupName) throws IOException {
    if (servers == null) {
        throw new ConstraintException("The list of servers to move cannot be null.");
    }
    if (servers.isEmpty()) {
        // TODO. Why? Stuff breaks if I equate them.
        return;
    }
    RSGroupInfo targetGrp = getAndCheckRSGroupInfo(targetGroupName);
    // another writer changing our state while we are working.
    synchronized (rsGroupInfoManager) {
        if (master.getMasterCoprocessorHost() != null) {
            master.getMasterCoprocessorHost().preMoveServers(servers, targetGroupName);
        }
        // Presume first server's source group. Later ensure all servers are from this group.
        Address firstServer = servers.iterator().next();
        RSGroupInfo srcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
        if (srcGrp == null) {
            // Be careful. This exception message is tested for in TestRSGroupsBase...
            throw new ConstraintException("Source RSGroup for server " + firstServer + " does not exist.");
        }
        if (srcGrp.getName().equals(targetGroupName)) {
            throw new ConstraintException("Target RSGroup " + targetGroupName + " is same as source " + srcGrp + " RSGroup.");
        }
        // groups. This prevents bogus servers from entering groups
        if (RSGroupInfo.DEFAULT_GROUP.equals(srcGrp.getName())) {
            checkOnlineServersOnly(servers);
        }
        // Ensure all servers are of same rsgroup.
        for (Address server : servers) {
            String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
            if (!tmpGroup.equals(srcGrp.getName())) {
                throw new ConstraintException("Move server request should only come from one source " + "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
            }
        }
        if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > 0) {
            throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() + " that contains tables without servers to host them.");
        }
        // MovedServers may be < passed in 'servers'.
        Set<Address> movedServers = rsGroupInfoManager.moveServers(servers, srcGrp.getName(), targetGroupName);
        List<Address> editableMovedServers = Lists.newArrayList(movedServers);
        boolean foundRegionsToUnassign;
        do {
            foundRegionsToUnassign = false;
            for (Iterator<Address> iter = editableMovedServers.iterator(); iter.hasNext(); ) {
                Address rs = iter.next();
                // Get regions that are associated with this server.
                List<HRegionInfo> regions = getRegions(rs);
                // Unassign regions for a server
                // TODO: This is problematic especially if hbase:meta is in the mix.
                // We need to update state in hbase:meta on Master and if unassigned we hang
                // around in here. There is a silly sort on linked list done above
                // in getRegions putting hbase:meta last which helps but probably has holes.
                LOG.info("Unassigning " + regions.size() + " region(s) from " + rs + " for server move to " + targetGroupName);
                if (!regions.isEmpty()) {
                    // TODO bulk unassign or throttled unassign?
                    for (HRegionInfo region : regions) {
                        // Regions might get assigned from tables of target group so we need to filter
                        if (!targetGrp.containsTable(region.getTable())) {
                            this.master.getAssignmentManager().unassign(region);
                            if (master.getAssignmentManager().getRegionStates().getRegionState(region).isFailedOpen()) {
                                // than mark region as 'foundRegionsToUnassign'.
                                continue;
                            }
                            foundRegionsToUnassign = true;
                        }
                    }
                }
                if (!foundRegionsToUnassign) {
                    iter.remove();
                }
            }
            try {
                rsGroupInfoManager.wait(1000);
            } catch (InterruptedException e) {
                LOG.warn("Sleep interrupted", e);
                Thread.currentThread().interrupt();
            }
        } while (foundRegionsToUnassign);
        if (master.getMasterCoprocessorHost() != null) {
            master.getMasterCoprocessorHost().postMoveServers(servers, targetGroupName);
        }
        LOG.info("Move server done: " + srcGrp.getName() + "=>" + targetGroupName);
    }
}

Example 9 with Address

use of org.apache.hadoop.hbase.net.Address in project hbase by apache.

the class RSGroupAdminServer method checkServersAndTables.

/**
   * Check servers and tables.
   * Fail if nulls or if servers and tables not belong to the same group
   * @param servers servers to move
   * @param tables tables to move
   * @param targetGroupName target group name
   * @throws IOException
   */
private void checkServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroupName) throws IOException {
    // Presume first server's source group. Later ensure all servers are from this group.
    Address firstServer = servers.iterator().next();
    RSGroupInfo tmpSrcGrp = rsGroupInfoManager.getRSGroupOfServer(firstServer);
    if (tmpSrcGrp == null) {
        // Be careful. This exception message is tested for in TestRSGroupsBase...
        throw new ConstraintException("Source RSGroup for server " + firstServer + " does not exist.");
    }
    RSGroupInfo srcGrp = new RSGroupInfo(tmpSrcGrp);
    if (srcGrp.getName().equals(targetGroupName)) {
        throw new ConstraintException("Target RSGroup " + targetGroupName + " is same as source " + srcGrp.getName() + " RSGroup.");
    }
    // Only move online servers
    checkOnlineServersOnly(servers);
    // Ensure all servers are of same rsgroup.
    for (Address server : servers) {
        String tmpGroup = rsGroupInfoManager.getRSGroupOfServer(server).getName();
        if (!tmpGroup.equals(srcGrp.getName())) {
            throw new ConstraintException("Move server request should only come from one source " + "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
        }
    }
    // Ensure all tables and servers are of same rsgroup.
    for (TableName table : tables) {
        String tmpGroup = rsGroupInfoManager.getRSGroupOfTable(table);
        if (!tmpGroup.equals(srcGrp.getName())) {
            throw new ConstraintException("Move table request should only come from one source " + "RSGroup. Expecting only " + srcGrp.getName() + " but contains " + tmpGroup);
        }
    }
    if (srcGrp.getServers().size() <= servers.size() && srcGrp.getTables().size() > tables.size()) {
        throw new ConstraintException("Cannot leave a RSGroup " + srcGrp.getName() + " that contains tables without servers to host them.");
    }
}

Example 10 with Address

use of org.apache.hadoop.hbase.net.Address in project hbase by apache.

the class RSGroupBasedLoadBalancer method balanceCluster.

@Override
public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) throws HBaseIOException {
    if (!isOnline()) {
        throw new ConstraintException(RSGroupInfoManager.RSGROUP_TABLE_NAME + " is not online, unable to perform balance");
    }
    Map<ServerName, List<HRegionInfo>> correctedState = correctAssignments(clusterState);
    List<RegionPlan> regionPlans = new ArrayList<>();
    List<HRegionInfo> misplacedRegions = correctedState.get(LoadBalancer.BOGUS_SERVER_NAME);
    for (HRegionInfo regionInfo : misplacedRegions) {
        regionPlans.add(new RegionPlan(regionInfo, null, null));
    }
    try {
        List<RSGroupInfo> rsgi = rsGroupInfoManager.listRSGroups();
        for (RSGroupInfo info : rsgi) {
            Map<ServerName, List<HRegionInfo>> groupClusterState = new HashMap<>();
            Map<TableName, Map<ServerName, List<HRegionInfo>>> groupClusterLoad = new HashMap<>();
            for (Address sName : info.getServers()) {
                for (ServerName curr : clusterState.keySet()) {
                    if (curr.getAddress().equals(sName)) {
                        groupClusterState.put(curr, correctedState.get(curr));
                    }
                }
            }
            groupClusterLoad.put(TableName.valueOf(HConstants.ENSEMBLE_TABLE_NAME), groupClusterState);
            this.internalBalancer.setClusterLoad(groupClusterLoad);
            List<RegionPlan> groupPlans = this.internalBalancer.balanceCluster(groupClusterState);
            if (groupPlans != null) {
                regionPlans.addAll(groupPlans);
            }
        }
    } catch (IOException exp) {
        LOG.warn("Exception while balancing cluster.", exp);
        regionPlans.clear();
    }
    return regionPlans;
}