Examples with RegionStates org.apache.hadoop.hbase.master.assignment.RegionStates used on opensource projects

Example 21 with RegionStates

use of org.apache.hadoop.hbase.master.assignment.RegionStates in project hbase by apache.

the class AbstractTestDLS method findRSToKill.

/**
 * Find a RS that has regions of a table.
 * @param hasMetaRegion when true, the returned RS has hbase:meta region as well
 */
private HRegionServer findRSToKill(boolean hasMetaRegion) throws Exception {
    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
    List<RegionInfo> regions = null;
    HRegionServer hrs = null;
    for (RegionServerThread rst : rsts) {
        hrs = rst.getRegionServer();
        while (rst.isAlive() && !hrs.isOnline()) {
            Thread.sleep(100);
        }
        if (!rst.isAlive()) {
            continue;
        }
        boolean isCarryingMeta = false;
        boolean foundTableRegion = false;
        regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
        for (RegionInfo region : regions) {
            if (region.isMetaRegion()) {
                isCarryingMeta = true;
            }
            if (region.getTable() == tableName) {
                foundTableRegion = true;
            }
            if (foundTableRegion && (isCarryingMeta || !hasMetaRegion)) {
                break;
            }
        }
        if (isCarryingMeta && hasMetaRegion) {
            // clients ask for a RS with META
            if (!foundTableRegion) {
                HRegionServer destRS = hrs;
                // the RS doesn't have regions of the specified table so we need move one to this RS
                List<RegionInfo> tableRegions = TEST_UTIL.getAdmin().getRegions(tableName);
                RegionInfo hri = tableRegions.get(0);
                TEST_UTIL.getAdmin().move(hri.getEncodedNameAsBytes(), destRS.getServerName());
                // wait for region move completes
                RegionStates regionStates = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
                TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {

                    @Override
                    public boolean evaluate() throws Exception {
                        ServerName sn = regionStates.getRegionServerOfRegion(hri);
                        return (sn != null && sn.equals(destRS.getServerName()));
                    }
                });
            }
            return hrs;
        } else if (hasMetaRegion || isCarryingMeta) {
            continue;
        }
        if (foundTableRegion) {
            break;
        }
    }
    return hrs;
}

Example 22 with RegionStates

use of org.apache.hadoop.hbase.master.assignment.RegionStates in project hbase by apache.

the class TestMetaFixer method testOverlapWithSmallMergeCount.

@Test
public void testOverlapWithSmallMergeCount() throws Exception {
    TableName tn = TableName.valueOf(this.name.getMethodName());
    try {
        testOverlapCommon(tn);
        HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
        CatalogJanitor cj = services.getCatalogJanitor();
        cj.scan();
        Report report = cj.getLastReport();
        assertEquals(6, report.getOverlaps().size());
        assertEquals(2, MetaFixer.calculateMerges(5, report.getOverlaps()).size());
        // The max merge count is set to 5 so overlap regions are divided into
        // two merge requests.
        TEST_UTIL.getHBaseCluster().getMaster().getConfiguration().setInt("hbase.master.metafixer.max.merge.count", 5);
        // Get overlap regions
        HashSet<String> overlapRegions = new HashSet<>();
        for (Pair<RegionInfo, RegionInfo> pair : report.getOverlaps()) {
            overlapRegions.add(pair.getFirst().getRegionNameAsString());
            overlapRegions.add(pair.getSecond().getRegionNameAsString());
        }
        MetaFixer fixer = new MetaFixer(services);
        fixer.fixOverlaps(report);
        AssignmentManager am = services.getAssignmentManager();
        HBaseTestingUtil.await(200, () -> {
            try {
                cj.scan();
                final Report postReport = cj.getLastReport();
                RegionStates regionStates = am.getRegionStates();
                RegionStateStore regionStateStore = am.getRegionStateStore();
                // Make sure that two merged regions are opened and GCs are done.
                if (postReport.getOverlaps().size() == 1) {
                    Pair<RegionInfo, RegionInfo> pair = postReport.getOverlaps().get(0);
                    if ((!overlapRegions.contains(pair.getFirst().getRegionNameAsString()) && regionStates.getRegionState(pair.getFirst()).isOpened()) && (!overlapRegions.contains(pair.getSecond().getRegionNameAsString()) && regionStates.getRegionState(pair.getSecond()).isOpened())) {
                        // Make sure GC is done.
                        List<RegionInfo> firstParents = regionStateStore.getMergeRegions(pair.getFirst());
                        List<RegionInfo> secondParents = regionStateStore.getMergeRegions(pair.getSecond());
                        return (firstParents == null || firstParents.isEmpty()) && (secondParents == null || secondParents.isEmpty());
                    }
                }
                return false;
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Second run of fixOverlap should fix all.
        report = cj.getLastReport();
        fixer.fixOverlaps(report);
        HBaseTestingUtil.await(20, () -> {
            try {
                // Make sure it GC only once.
                return (cj.scan() > 0);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // No holes reported.
        cj.scan();
        final Report postReport = cj.getLastReport();
        assertTrue(postReport.isEmpty());
    } finally {
        TEST_UTIL.getHBaseCluster().getMaster().getConfiguration().unset("hbase.master.metafixer.max.merge.count");
        TEST_UTIL.deleteTable(tn);
    }
}

Example 23 with RegionStates

use of org.apache.hadoop.hbase.master.assignment.RegionStates in project hbase by apache.

the class TestFavoredStochasticBalancerPickers method testPickers.

@Test
public void testPickers() throws Exception {
    TableName tableName = TableName.valueOf(name.getMethodName());
    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY).build();
    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(columnFamilyDescriptor).build();
    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGIONS);
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY);
    admin.flush(tableName);
    HMaster master = cluster.getMaster();
    FavoredNodesManager fnm = master.getFavoredNodesManager();
    ServerName masterServerName = master.getServerName();
    List<ServerName> excludedServers = Lists.newArrayList(masterServerName);
    final ServerName mostLoadedServer = getRSWithMaxRegions(tableName, excludedServers);
    assertNotNull(mostLoadedServer);
    int numRegions = getTableRegionsFromServer(tableName, mostLoadedServer).size();
    excludedServers.add(mostLoadedServer);
    // Lets find another server with more regions to calculate number of regions to move
    ServerName source = getRSWithMaxRegions(tableName, excludedServers);
    assertNotNull(source);
    int regionsToMove = getTableRegionsFromServer(tableName, source).size() / 2;
    // Since move only works if the target is part of favored nodes of the region, lets get all
    // regions that are movable to mostLoadedServer
    List<RegionInfo> hris = getRegionsThatCanBeMoved(tableName, mostLoadedServer);
    RegionStates rst = master.getAssignmentManager().getRegionStates();
    for (int i = 0; i < regionsToMove; i++) {
        final RegionInfo regionInfo = hris.get(i);
        admin.move(regionInfo.getEncodedNameAsBytes(), mostLoadedServer);
        LOG.info("Moving region: " + hris.get(i).getRegionNameAsString() + " to " + mostLoadedServer);
        TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                return ServerName.isSameAddress(rst.getRegionServerOfRegion(regionInfo), mostLoadedServer);
            }
        });
    }
    final int finalRegions = numRegions + regionsToMove;
    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
    TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            int numRegions = getTableRegionsFromServer(tableName, mostLoadedServer).size();
            return (numRegions == finalRegions);
        }
    });
    TEST_UTIL.getHBaseCluster().startRegionServerAndWait(60000);
    Map<ServerName, List<RegionInfo>> serverAssignments = Maps.newHashMap();
    ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
        if (!ServerName.isSameAddress(sn, masterServerName)) {
            serverAssignments.put(sn, getTableRegionsFromServer(tableName, sn));
        }
    }
    RegionHDFSBlockLocationFinder regionFinder = new RegionHDFSBlockLocationFinder();
    regionFinder.setClusterMetrics(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)));
    regionFinder.setConf(conf);
    regionFinder.setClusterInfoProvider(new MasterClusterInfoProvider(TEST_UTIL.getMiniHBaseCluster().getMaster()));
    BalancerClusterState cluster = new BalancerClusterState(serverAssignments, null, regionFinder, new RackManager(conf));
    LoadOnlyFavoredStochasticBalancer balancer = (LoadOnlyFavoredStochasticBalancer) TEST_UTIL.getMiniHBaseCluster().getMaster().getLoadBalancer().getInternalBalancer();
    cluster.sortServersByRegionCount();
    Integer[] servers = cluster.serverIndicesSortedByRegionCount;
    LOG.info("Servers sorted by region count:" + Arrays.toString(servers));
    LOG.info("Cluster dump: " + cluster);
    if (!mostLoadedServer.equals(cluster.servers[servers[servers.length - 1]])) {
        LOG.error("Most loaded server: " + mostLoadedServer + " does not match: " + cluster.servers[servers[servers.length - 1]]);
    }
    assertEquals(mostLoadedServer, cluster.servers[servers[servers.length - 1]]);
    FavoredStochasticBalancer.FavoredNodeLoadPicker loadPicker = balancer.new FavoredNodeLoadPicker();
    boolean userRegionPicked = false;
    for (int i = 0; i < 100; i++) {
        if (userRegionPicked) {
            break;
        } else {
            BalanceAction action = loadPicker.generate(cluster);
            if (action.getType() == BalanceAction.Type.MOVE_REGION) {
                MoveRegionAction moveRegionAction = (MoveRegionAction) action;
                RegionInfo region = cluster.regions[moveRegionAction.getRegion()];
                assertNotEquals(-1, moveRegionAction.getToServer());
                ServerName destinationServer = cluster.servers[moveRegionAction.getToServer()];
                assertEquals(cluster.servers[moveRegionAction.getFromServer()], mostLoadedServer);
                if (!region.getTable().isSystemTable()) {
                    List<ServerName> favNodes = fnm.getFavoredNodes(region);
                    assertTrue(favNodes.contains(ServerName.valueOf(destinationServer.getAddress(), -1)));
                    userRegionPicked = true;
                }
            }
        }
    }
    assertTrue("load picker did not pick expected regions in 100 iterations.", userRegionPicked);
}

Example 24 with RegionStates

use of org.apache.hadoop.hbase.master.assignment.RegionStates in project hbase by apache.

the class TestFavoredStochasticBalancerPickers method getRegionsThatCanBeMoved.

/*
   * A region can only be moved to one of its favored node. Hence this method helps us to
   * get that list which makes it easy to write non-flaky tests.
   */
private List<RegionInfo> getRegionsThatCanBeMoved(TableName tableName, ServerName serverName) {
    List<RegionInfo> regions = Lists.newArrayList();
    RegionStates rst = cluster.getMaster().getAssignmentManager().getRegionStates();
    FavoredNodesManager fnm = cluster.getMaster().getFavoredNodesManager();
    for (RegionInfo regionInfo : fnm.getRegionsOfFavoredNode(serverName)) {
        if (regionInfo.getTable().equals(tableName) && !ServerName.isSameAddress(rst.getRegionServerOfRegion(regionInfo), serverName)) {
            regions.add(regionInfo);
        }
    }
    return regions;
}

Example 25 with RegionStates

use of org.apache.hadoop.hbase.master.assignment.RegionStates in project hbase by apache.

the class TestFavoredStochasticLoadBalancer method test2FavoredNodesDead.

@Test
public void test2FavoredNodesDead() throws Exception {
    TableName tableName = TableName.valueOf("testAllFavoredNodesDead");
    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
    admin.createTable(tableDescriptor, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
    TEST_UTIL.waitTableAvailable(tableName);
    final RegionInfo region = admin.getRegions(tableName).get(0);
    LOG.info("Region that's supposed to be in transition: " + region);
    FavoredNodesManager fnm = master.getFavoredNodesManager();
    List<ServerName> currentFN = fnm.getFavoredNodes(region);
    assertNotNull(currentFN);
    List<ServerName> serversToStop = Lists.newArrayList(currentFN);
    serversToStop.remove(currentFN.get(0));
    // Lets kill 2 FN for the region. All regions should still be assigned
    stopServersAndWaitUntilProcessed(serversToStop);
    TEST_UTIL.waitUntilNoRegionsInTransition();
    final RegionStates regionStates = master.getAssignmentManager().getRegionStates();
    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return regionStates.getRegionState(region).isOpened();
        }
    });
    assertEquals("Not all regions are online", REGION_NUM, admin.getRegions(tableName).size());
    admin.balancerSwitch(true, true);
    assertTrue("Balancer did not run", admin.balance());
    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
    checkFavoredNodeAssignments(tableName, fnm, regionStates);
}