Examples with FavoredNodesManager org.apache.hadoop.hbase.favored.FavoredNodesManager used on opensource projects

Example 11 with FavoredNodesManager

use of org.apache.hadoop.hbase.favored.FavoredNodesManager in project hbase by apache.

the class TestFavoredStochasticLoadBalancer method testAllFavoredNodesDeadMasterRestarted.

@Ignore
@Test
public void testAllFavoredNodesDeadMasterRestarted() throws Exception {
    TableName tableName = TableName.valueOf("testAllFavoredNodesDeadMasterRestarted");
    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.of(HConstants.CATALOG_FAMILY)).build();
    admin.createTable(tableDescriptor, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
    TEST_UTIL.waitTableAvailable(tableName);
    final RegionInfo region = admin.getRegions(tableName).get(0);
    LOG.info("Region that's supposed to be in transition: " + region);
    FavoredNodesManager fnm = master.getFavoredNodesManager();
    List<ServerName> currentFN = fnm.getFavoredNodes(region);
    assertNotNull(currentFN);
    // Lets kill all the RS that are favored nodes for this region.
    stopServersAndWaitUntilProcessed(currentFN);
    final RegionStates regionStatesBeforeMaster = master.getAssignmentManager().getRegionStates();
    TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            return regionStatesBeforeMaster.getRegionState(region).isFailedOpen();
        }
    });
    assertTrue("Region: " + region + " should be RIT", regionStatesBeforeMaster.getRegionState(region).isFailedOpen());
    List<RegionInfo> rit = Lists.newArrayList();
    for (RegionStateNode regionState : regionStatesBeforeMaster.getRegionsInTransition()) {
        RegionInfo regionInfo = regionState.getRegionInfo();
        LOG.debug("Region in transition after stopping FN's: " + regionInfo);
        rit.add(regionInfo);
        assertTrue("Region: " + regionInfo + " should be RIT", regionStatesBeforeMaster.getRegionState(regionInfo).isFailedOpen());
        assertEquals("Region: " + regionInfo + " does not belong to table: " + tableName, tableName, regionInfo.getTable());
    }
    Configuration conf = cluster.getConf();
    conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, SLAVES - FavoredNodeAssignmentHelper.FAVORED_NODES_NUM);
    cluster.stopMaster(master.getServerName());
    cluster.waitForMasterToStop(master.getServerName(), 60000);
    cluster.startMaster();
    cluster.waitForActiveAndReadyMaster();
    master = cluster.getMaster();
    fnm = master.getFavoredNodesManager();
    RegionStates regionStates = master.getAssignmentManager().getRegionStates();
    assertTrue("Region: " + region + " should be RIT", regionStates.getRegionState(region).isFailedOpen());
    for (RegionInfo regionInfo : rit) {
        assertTrue("Region: " + regionInfo + " should be RIT", regionStates.getRegionState(regionInfo).isFailedOpen());
    }
    // Regenerate FN and assign, everything else should be fine
    List<ServerName> serversForNewFN = Lists.newArrayList();
    for (ServerName sn : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet()) {
        serversForNewFN.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), NON_STARTCODE));
    }
    FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(serversForNewFN, conf);
    helper.initialize();
    for (RegionInfo regionInfo : rit) {
        List<ServerName> newFavoredNodes = helper.generateFavoredNodes(regionInfo);
        assertNotNull(newFavoredNodes);
        assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, newFavoredNodes.size());
        LOG.info("Region: " + regionInfo.getEncodedName() + " FN: " + newFavoredNodes);
        Map<RegionInfo, List<ServerName>> regionFNMap = Maps.newHashMap();
        regionFNMap.put(regionInfo, newFavoredNodes);
        fnm.updateFavoredNodes(regionFNMap);
        LOG.info("Assigning region: " + regionInfo.getEncodedName());
        admin.assign(regionInfo.getEncodedNameAsBytes());
    }
    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
    assertEquals("Not all regions are online", REGION_NUM, admin.getRegions(tableName).size());
    admin.balancerSwitch(true, true);
    assertTrue("Balancer did not run", admin.balance());
    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
    checkFavoredNodeAssignments(tableName, fnm, regionStates);
}

Example 12 with FavoredNodesManager

use of org.apache.hadoop.hbase.favored.FavoredNodesManager in project hbase by apache.

the class CatalogJanitor method cleanParent.

/**
   * If daughters no longer hold reference to the parents, delete the parent.
   * @param parent HRegionInfo of split offlined parent
   * @param rowContent Content of <code>parent</code> row in
   * <code>metaRegionName</code>
   * @return True if we removed <code>parent</code> from meta table and from
   * the filesystem.
   * @throws IOException
   */
boolean cleanParent(final HRegionInfo parent, Result rowContent) throws IOException {
    boolean result = false;
    // be inserted/deleted together
    if (rowContent.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
        // wait cleaning merge region first
        return result;
    }
    // Run checks on each daughter split.
    PairOfSameType<HRegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
    Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
    Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
    if (hasNoReferences(a) && hasNoReferences(b)) {
        LOG.debug("Deleting region " + parent.getRegionNameAsString() + " because daughter splits no longer hold references");
        FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
        if (LOG.isTraceEnabled())
            LOG.trace("Archiving parent region: " + parent);
        HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
        MetaTableAccessor.deleteRegion(this.connection, parent);
        services.getServerManager().removeRegion(parent);
        FavoredNodesManager fnm = this.services.getFavoredNodesManager();
        if (fnm != null) {
            fnm.deleteFavoredNodesForRegions(Lists.newArrayList(parent));
        }
        result = true;
    }
    return result;
}

Example 13 with FavoredNodesManager

use of org.apache.hadoop.hbase.favored.FavoredNodesManager in project hbase by apache.

the class AssignmentManagerUtil method removeNonDefaultReplicas.

static void removeNonDefaultReplicas(MasterProcedureEnv env, Stream<RegionInfo> regions, int regionReplication) {
    // Remove from in-memory states
    regions.flatMap(hri -> IntStream.range(1, regionReplication).mapToObj(i -> RegionReplicaUtil.getRegionInfoForReplica(hri, i))).forEach(hri -> {
        env.getAssignmentManager().getRegionStates().deleteRegion(hri);
        env.getMasterServices().getServerManager().removeRegion(hri);
        FavoredNodesManager fnm = env.getMasterServices().getFavoredNodesManager();
        if (fnm != null) {
            fnm.deleteFavoredNodesForRegions(Collections.singletonList(hri));
        }
    });
}

Example 14 with FavoredNodesManager

use of org.apache.hadoop.hbase.favored.FavoredNodesManager in project hbase by apache.

the class RSGroupBasedLoadBalancer method initialize.

@Override
public void initialize() throws IOException {
    if (rsGroupInfoManager == null) {
        rsGroupInfoManager = masterServices.getRSGroupInfoManager();
        if (rsGroupInfoManager == null) {
            String msg = "RSGroupInfoManager hasn't been initialized";
            LOG.error(msg);
            throw new HBaseIOException(msg);
        }
        rsGroupInfoManager.start();
    }
    // Create the balancer
    Configuration conf = masterServices.getConfiguration();
    Class<? extends LoadBalancer> balancerClass;
    @SuppressWarnings("deprecation") String balancerClassName = conf.get(HBASE_RSGROUP_LOADBALANCER_CLASS);
    if (balancerClassName == null) {
        balancerClass = conf.getClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory.getDefaultLoadBalancerClass(), LoadBalancer.class);
    } else {
        try {
            balancerClass = Class.forName(balancerClassName).asSubclass(LoadBalancer.class);
        } catch (ClassNotFoundException e) {
            throw new IOException(e);
        }
    }
    this.provider = new MasterClusterInfoProvider(masterServices);
    // avoid infinite nesting
    if (getClass().isAssignableFrom(balancerClass)) {
        balancerClass = LoadBalancerFactory.getDefaultLoadBalancerClass();
    }
    internalBalancer = ReflectionUtils.newInstance(balancerClass);
    internalBalancer.setClusterInfoProvider(provider);
    // special handling for favor node balancers
    if (internalBalancer instanceof FavoredNodesPromoter) {
        favoredNodesManager = new FavoredNodesManager(provider);
        ((FavoredNodesPromoter) internalBalancer).setFavoredNodesManager(favoredNodesManager);
    }
    internalBalancer.initialize();
    // init fallback groups
    this.fallbackEnabled = conf.getBoolean(FALLBACK_GROUP_ENABLE_KEY, false);
}

Example 15 with FavoredNodesManager

use of org.apache.hadoop.hbase.favored.FavoredNodesManager in project hbase by apache.

the class TestFavoredStochasticBalancerPickers method testPickers.

@Test
public void testPickers() throws Exception {
    TableName tableName = TableName.valueOf(name.getMethodName());
    ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY).build();
    TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(columnFamilyDescriptor).build();
    admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGIONS);
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY);
    admin.flush(tableName);
    HMaster master = cluster.getMaster();
    FavoredNodesManager fnm = master.getFavoredNodesManager();
    ServerName masterServerName = master.getServerName();
    List<ServerName> excludedServers = Lists.newArrayList(masterServerName);
    final ServerName mostLoadedServer = getRSWithMaxRegions(tableName, excludedServers);
    assertNotNull(mostLoadedServer);
    int numRegions = getTableRegionsFromServer(tableName, mostLoadedServer).size();
    excludedServers.add(mostLoadedServer);
    // Lets find another server with more regions to calculate number of regions to move
    ServerName source = getRSWithMaxRegions(tableName, excludedServers);
    assertNotNull(source);
    int regionsToMove = getTableRegionsFromServer(tableName, source).size() / 2;
    // Since move only works if the target is part of favored nodes of the region, lets get all
    // regions that are movable to mostLoadedServer
    List<RegionInfo> hris = getRegionsThatCanBeMoved(tableName, mostLoadedServer);
    RegionStates rst = master.getAssignmentManager().getRegionStates();
    for (int i = 0; i < regionsToMove; i++) {
        final RegionInfo regionInfo = hris.get(i);
        admin.move(regionInfo.getEncodedNameAsBytes(), mostLoadedServer);
        LOG.info("Moving region: " + hris.get(i).getRegionNameAsString() + " to " + mostLoadedServer);
        TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                return ServerName.isSameAddress(rst.getRegionServerOfRegion(regionInfo), mostLoadedServer);
            }
        });
    }
    final int finalRegions = numRegions + regionsToMove;
    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
    TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            int numRegions = getTableRegionsFromServer(tableName, mostLoadedServer).size();
            return (numRegions == finalRegions);
        }
    });
    TEST_UTIL.getHBaseCluster().startRegionServerAndWait(60000);
    Map<ServerName, List<RegionInfo>> serverAssignments = Maps.newHashMap();
    ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
    for (ServerName sn : status.getLiveServerMetrics().keySet()) {
        if (!ServerName.isSameAddress(sn, masterServerName)) {
            serverAssignments.put(sn, getTableRegionsFromServer(tableName, sn));
        }
    }
    RegionHDFSBlockLocationFinder regionFinder = new RegionHDFSBlockLocationFinder();
    regionFinder.setClusterMetrics(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)));
    regionFinder.setConf(conf);
    regionFinder.setClusterInfoProvider(new MasterClusterInfoProvider(TEST_UTIL.getMiniHBaseCluster().getMaster()));
    BalancerClusterState cluster = new BalancerClusterState(serverAssignments, null, regionFinder, new RackManager(conf));
    LoadOnlyFavoredStochasticBalancer balancer = (LoadOnlyFavoredStochasticBalancer) TEST_UTIL.getMiniHBaseCluster().getMaster().getLoadBalancer().getInternalBalancer();
    cluster.sortServersByRegionCount();
    Integer[] servers = cluster.serverIndicesSortedByRegionCount;
    LOG.info("Servers sorted by region count:" + Arrays.toString(servers));
    LOG.info("Cluster dump: " + cluster);
    if (!mostLoadedServer.equals(cluster.servers[servers[servers.length - 1]])) {
        LOG.error("Most loaded server: " + mostLoadedServer + " does not match: " + cluster.servers[servers[servers.length - 1]]);
    }
    assertEquals(mostLoadedServer, cluster.servers[servers[servers.length - 1]]);
    FavoredStochasticBalancer.FavoredNodeLoadPicker loadPicker = balancer.new FavoredNodeLoadPicker();
    boolean userRegionPicked = false;
    for (int i = 0; i < 100; i++) {
        if (userRegionPicked) {
            break;
        } else {
            BalanceAction action = loadPicker.generate(cluster);
            if (action.getType() == BalanceAction.Type.MOVE_REGION) {
                MoveRegionAction moveRegionAction = (MoveRegionAction) action;
                RegionInfo region = cluster.regions[moveRegionAction.getRegion()];
                assertNotEquals(-1, moveRegionAction.getToServer());
                ServerName destinationServer = cluster.servers[moveRegionAction.getToServer()];
                assertEquals(cluster.servers[moveRegionAction.getFromServer()], mostLoadedServer);
                if (!region.getTable().isSystemTable()) {
                    List<ServerName> favNodes = fnm.getFavoredNodes(region);
                    assertTrue(favNodes.contains(ServerName.valueOf(destinationServer.getAddress(), -1)));
                    userRegionPicked = true;
                }
            }
        }
    }
    assertTrue("load picker did not pick expected regions in 100 iterations.", userRegionPicked);
}