Examples with RSGroupBasedLoadBalancer org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer used on opensource projects

Example 1 with RSGroupBasedLoadBalancer

use of org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer in project hbase by apache.

the class HMaster method initializeZKBasedSystemTrackers.

/**
 * Initialize all ZK based system trackers. But do not include {@link RegionServerTracker}, it
 * should have already been initialized along with {@link ServerManager}.
 */
private void initializeZKBasedSystemTrackers() throws IOException, KeeperException, ReplicationException {
    if (maintenanceMode) {
        // in maintenance mode, always use MaintenanceLoadBalancer.
        conf.unset(LoadBalancer.HBASE_RSGROUP_LOADBALANCER_CLASS);
        conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MaintenanceLoadBalancer.class, LoadBalancer.class);
    }
    this.balancer = new RSGroupBasedLoadBalancer();
    this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this);
    this.loadBalancerTracker.start();
    this.regionNormalizerManager = RegionNormalizerFactory.createNormalizerManager(conf, zooKeeper, this);
    this.configurationManager.registerObserver(regionNormalizerManager);
    this.regionNormalizerManager.start();
    this.splitOrMergeTracker = new SplitOrMergeTracker(zooKeeper, conf, this);
    this.splitOrMergeTracker.start();
    // This is for backwards compatible. We do not need the CP for rs group now but if user want to
    // load it, we need to enable rs group.
    String[] cpClasses = conf.getStrings(MasterCoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
    if (cpClasses != null) {
        for (String cpClass : cpClasses) {
            if (RSGroupAdminEndpoint.class.getName().equals(cpClass)) {
                RSGroupUtil.enableRSGroup(conf);
                break;
            }
        }
    }
    this.rsGroupInfoManager = RSGroupInfoManager.create(this);
    this.replicationPeerManager = ReplicationPeerManager.create(zooKeeper, conf, clusterId);
    this.drainingServerTracker = new DrainingServerTracker(zooKeeper, this, this.serverManager);
    this.drainingServerTracker.start();
    this.snapshotCleanupTracker = new SnapshotCleanupTracker(zooKeeper, this);
    this.snapshotCleanupTracker.start();
    String clientQuorumServers = conf.get(HConstants.CLIENT_ZOOKEEPER_QUORUM);
    boolean clientZkObserverMode = conf.getBoolean(HConstants.CLIENT_ZOOKEEPER_OBSERVER_MODE, HConstants.DEFAULT_CLIENT_ZOOKEEPER_OBSERVER_MODE);
    if (clientQuorumServers != null && !clientZkObserverMode) {
        // we need to take care of the ZK information synchronization
        // if given client ZK are not observer nodes
        ZKWatcher clientZkWatcher = new ZKWatcher(conf, getProcessName() + ":" + rpcServices.getSocketAddress().getPort() + "-clientZK", this, false, true);
        this.metaLocationSyncer = new MetaLocationSyncer(zooKeeper, clientZkWatcher, this);
        this.metaLocationSyncer.start();
        this.masterAddressSyncer = new MasterAddressSyncer(zooKeeper, clientZkWatcher, this);
        this.masterAddressSyncer.start();
        // set cluster id is a one-go effort
        ZKClusterId.setClusterId(clientZkWatcher, fileSystemManager.getClusterId());
    }
    // Set the cluster as up.  If new RSs, they'll be waiting on this before
    // going ahead with their startup.
    boolean wasUp = this.clusterStatusTracker.isClusterUp();
    if (!wasUp)
        this.clusterStatusTracker.setClusterUp();
    LOG.info("Active/primary master=" + this.serverName + ", sessionid=0x" + Long.toHexString(this.zooKeeper.getRecoverableZooKeeper().getSessionId()) + ", setting cluster-up flag (Was=" + wasUp + ")");
    // create/initialize the snapshot manager and other procedure managers
    this.snapshotManager = new SnapshotManager();
    this.mpmHost = new MasterProcedureManagerHost();
    this.mpmHost.register(this.snapshotManager);
    this.mpmHost.register(new MasterFlushTableProcedureManager());
    this.mpmHost.loadProcedures(conf);
    this.mpmHost.initialize(this, this.metricsMaster);
}

Example 2 with RSGroupBasedLoadBalancer

use of org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer in project hbase by apache.

the class TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal method beforeAllTests.

@BeforeClass
public static void beforeAllTests() throws Exception {
    groups = new String[] { RSGroupInfo.DEFAULT_GROUP };
    servers = generateServers(3);
    groupMap = constructGroupInfo(servers, groups);
    tableDescs = constructTableDesc(false);
    conf.set("hbase.regions.slop", "0");
    conf.setFloat("hbase.master.balancer.stochastic.readRequestCost", 10000f);
    conf.set("hbase.rsgroup.grouploadbalancer.class", StochasticLoadBalancer.class.getCanonicalName());
    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
    loadBalancer = new RSGroupBasedLoadBalancer();
    loadBalancer.setMasterServices(getMockedMaster());
    loadBalancer.initialize();
}

Example 3 with RSGroupBasedLoadBalancer

use of org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer in project hbase by apache.

the class TestMasterBalancerNPE method testBalancerNPE.

/**
 * This test is for HBASE-26712, to make the region is unassigned just before
 * {@link AssignmentManager#balance} is invoked on the region.
 */
@Test
public void testBalancerNPE() throws Exception {
    TEST_UTIL.startMiniCluster(2);
    TEST_UTIL.getAdmin().balancerSwitch(false, true);
    TableName tableName = createTable(name.getMethodName());
    final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
    List<RegionInfo> regionInfos = TEST_UTIL.getAdmin().getRegions(tableName);
    assertTrue(regionInfos.size() == 1);
    final ServerName serverName1 = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
    final ServerName serverName2 = TEST_UTIL.getMiniHBaseCluster().getRegionServer(1).getServerName();
    final RegionInfo regionInfo = regionInfos.get(0);
    RSGroupBasedLoadBalancer loadBalancer = master.getLoadBalancer();
    RSGroupBasedLoadBalancer spiedLoadBalancer = Mockito.spy(loadBalancer);
    final AtomicReference<RegionPlan> regionPlanRef = new AtomicReference<RegionPlan>();
    /**
     * Mock {@link RSGroupBasedLoadBalancer#balanceCluster} to return the {@link RegionPlan} to move
     * the only region to the other RegionServer.
     */
    Mockito.doAnswer((InvocationOnMock invocation) -> {
        @SuppressWarnings("unchecked") Map<TableName, Map<ServerName, List<RegionInfo>>> tableNameToRegionServerNameToRegionInfos = (Map<TableName, Map<ServerName, List<RegionInfo>>>) invocation.getArgument(0);
        Map<ServerName, List<RegionInfo>> regionServerNameToRegionInfos = tableNameToRegionServerNameToRegionInfos.get(tableName);
        assertTrue(regionServerNameToRegionInfos.size() == 2);
        List<ServerName> assignedRegionServerNames = new ArrayList<ServerName>();
        for (Map.Entry<ServerName, List<RegionInfo>> entry : regionServerNameToRegionInfos.entrySet()) {
            if (entry.getValue().size() > 0) {
                assignedRegionServerNames.add(entry.getKey());
            }
        }
        assertTrue(assignedRegionServerNames.size() == 1);
        ServerName assignedRegionServerName = assignedRegionServerNames.get(0);
        ServerName notAssignedRegionServerName = assignedRegionServerName.equals(serverName1) ? serverName2 : serverName1;
        RegionPlan regionPlan = new RegionPlan(regionInfo, assignedRegionServerName, notAssignedRegionServerName);
        regionPlanRef.set(regionPlan);
        return Arrays.asList(regionPlan);
    }).when(spiedLoadBalancer).balanceCluster(Mockito.anyMap());
    AssignmentManager assignmentManager = master.getAssignmentManager();
    final AssignmentManager spiedAssignmentManager = Mockito.spy(assignmentManager);
    final CyclicBarrier cyclicBarrier = new CyclicBarrier(2);
    /**
     * Override {@link AssignmentManager#balance} to invoke real {@link AssignmentManager#balance}
     * after the region is successfully unassigned.
     */
    Mockito.doAnswer((InvocationOnMock invocation) -> {
        RegionPlan regionPlan = invocation.getArgument(0);
        RegionPlan referedRegionPlan = regionPlanRef.get();
        assertTrue(referedRegionPlan != null);
        if (referedRegionPlan.equals(regionPlan)) {
            /**
             * To make {@link AssignmentManager#unassign} could be invoked just before
             * {@link AssignmentManager#balance} is invoked.
             */
            cyclicBarrier.await();
            /**
             * After {@link AssignmentManager#unassign} is completed,we could invoke
             * {@link AssignmentManager#balance}.
             */
            cyclicBarrier.await();
        }
        /**
         * Before HBASE-26712,here may throw NPE.
         */
        return invocation.callRealMethod();
    }).when(spiedAssignmentManager).balance(Mockito.any());
    try {
        final AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>(null);
        Thread unassignThread = new Thread(() -> {
            try {
                /**
                 * To invoke {@link AssignmentManager#unassign} just before
                 * {@link AssignmentManager#balance} is invoked.
                 */
                cyclicBarrier.await();
                spiedAssignmentManager.unassign(regionInfo);
                assertTrue(spiedAssignmentManager.getRegionStates().getRegionAssignments().get(regionInfo) == null);
                /**
                 * After {@link AssignmentManager#unassign} is completed,we could invoke
                 * {@link AssignmentManager#balance}.
                 */
                cyclicBarrier.await();
            } catch (Exception e) {
                exceptionRef.set(e);
            }
        });
        unassignThread.setName("UnassignThread");
        unassignThread.start();
        master.setLoadBalancer(spiedLoadBalancer);
        master.setAssignmentManager(spiedAssignmentManager);
        /**
         * enable balance
         */
        TEST_UTIL.getAdmin().balancerSwitch(true, false);
        /**
         * Before HBASE-26712,here invokes {@link AssignmentManager#balance(RegionPlan)}
         * which may throw NPE.
         */
        master.balanceOrUpdateMetrics();
        unassignThread.join();
        assertTrue(exceptionRef.get() == null);
    } finally {
        master.setLoadBalancer(loadBalancer);
        master.setAssignmentManager(assignmentManager);
    }
}

Example 4 with RSGroupBasedLoadBalancer

use of org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer in project hbase by apache.

the class TestRSGroupBasedLoadBalancer method beforeAllTests.

@BeforeClass
public static void beforeAllTests() throws Exception {
    servers = generateServers(7);
    groupMap = constructGroupInfo(servers, groups);
    tableDescs = constructTableDesc(true);
    conf.set("hbase.regions.slop", "0");
    conf.set("hbase.rsgroup.grouploadbalancer.class", SimpleLoadBalancer.class.getCanonicalName());
    conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
    loadBalancer = new RSGroupBasedLoadBalancer();
    loadBalancer.setMasterServices(getMockedMaster());
    loadBalancer.initialize();
}