Examples with RegionPlan org.apache.hadoop.hbase.master.RegionPlan used on opensource projects

Example 11 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class SimpleLoadBalancer method addRegionPlan.

/**
 * Add a region from the head or tail to the List of regions to return.
 */
private void addRegionPlan(final MinMaxPriorityQueue<RegionPlan> regionsToMove, final boolean fetchFromTail, final ServerName sn, List<RegionPlan> regionsToReturn) {
    RegionPlan rp = null;
    if (!fetchFromTail) {
        rp = regionsToMove.remove();
    } else {
        rp = regionsToMove.removeLast();
    }
    rp.setDestination(sn);
    regionsToReturn.add(rp);
}

Example 12 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class TestOpenRegionProcedureHang method test.

@Test
public void test() throws InterruptedException, KeeperException, IOException {
    RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
    AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
    HRegionServer rs1 = UTIL.getRSForFirstRegionInTable(NAME);
    HRegionServer rs2 = UTIL.getOtherRegionServer(rs1);
    ARRIVE = new CountDownLatch(1);
    RESUME = new CountDownLatch(1);
    FINISH = new CountDownLatch(1);
    ABORT = new CountDownLatch(1);
    am.moveAsync(new RegionPlan(region, rs1.getServerName(), rs2.getServerName()));
    ARRIVE.await();
    ARRIVE = null;
    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
    master.getZooKeeper().close();
    UTIL.waitFor(30000, () -> {
        for (MasterThread mt : UTIL.getMiniHBaseCluster().getMasterThreads()) {
            if (mt.getMaster() != master && mt.getMaster().isActiveMaster()) {
                return mt.getMaster().isInitialized();
            }
        }
        return false;
    });
    ProcedureExecutor<MasterProcedureEnv> procExec = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    UTIL.waitFor(30000, () -> procExec.getProcedures().stream().filter(p -> p instanceof OpenRegionProcedure).map(p -> (OpenRegionProcedure) p).anyMatch(p -> p.region.getTable().equals(NAME)));
    OpenRegionProcedure proc = procExec.getProcedures().stream().filter(p -> p instanceof OpenRegionProcedure).map(p -> (OpenRegionProcedure) p).filter(p -> p.region.getTable().equals(NAME)).findFirst().get();
    // wait a bit to let the OpenRegionProcedure send out the request
    Thread.sleep(2000);
    RESUME.countDown();
    if (!FINISH.await(15, TimeUnit.SECONDS)) {
        LOG.info("Wait reportRegionStateTransition to finish timed out, this is possible if" + " we update the procedure store, as the WALProcedureStore" + " will retry forever to roll the writer if it is not closed");
    }
    FINISH = null;
    // if the reportRegionTransition is finished, wait a bit to let it return the data to RS
    Thread.sleep(2000);
    ABORT.countDown();
    UTIL.waitFor(30000, () -> procExec.isFinished(proc.getProcId()));
    UTIL.waitFor(30000, () -> procExec.isFinished(proc.getParentProcId()));
}

Example 13 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal method testBalanceCluster.

/**
 * Test HBASE-20791
 */
@Test
public void testBalanceCluster() throws IOException {
    // mock cluster State
    Map<ServerName, List<RegionInfo>> clusterState = new HashMap<ServerName, List<RegionInfo>>();
    ServerName serverA = servers.get(0);
    ServerName serverB = servers.get(1);
    ServerName serverC = servers.get(2);
    List<RegionInfo> regionsOnServerA = randomRegions(3);
    List<RegionInfo> regionsOnServerB = randomRegions(3);
    List<RegionInfo> regionsOnServerC = randomRegions(3);
    clusterState.put(serverA, regionsOnServerA);
    clusterState.put(serverB, regionsOnServerB);
    clusterState.put(serverC, regionsOnServerC);
    // mock ClusterMetrics
    Map<ServerName, ServerMetrics> serverMetricsMap = new TreeMap<>();
    serverMetricsMap.put(serverA, mockServerMetricsWithReadRequests(serverA, regionsOnServerA, 0));
    serverMetricsMap.put(serverB, mockServerMetricsWithReadRequests(serverB, regionsOnServerB, 0));
    serverMetricsMap.put(serverC, mockServerMetricsWithReadRequests(serverC, regionsOnServerC, 0));
    ClusterMetrics clusterStatus = mock(ClusterMetrics.class);
    when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap);
    loadBalancer.updateClusterMetrics(clusterStatus);
    // ReadRequestCostFunction are Rate based, So doing setClusterMetrics again
    // this time, regions on serverA with more readRequestCount load
    // serverA : 1000,1000,1000
    // serverB : 0,0,0
    // serverC : 0,0,0
    // so should move two regions from serverA to serverB & serverC
    serverMetricsMap = new TreeMap<>();
    serverMetricsMap.put(serverA, mockServerMetricsWithReadRequests(serverA, regionsOnServerA, 1000));
    serverMetricsMap.put(serverB, mockServerMetricsWithReadRequests(serverB, regionsOnServerB, 0));
    serverMetricsMap.put(serverC, mockServerMetricsWithReadRequests(serverC, regionsOnServerC, 0));
    clusterStatus = mock(ClusterMetrics.class);
    when(clusterStatus.getLiveServerMetrics()).thenReturn(serverMetricsMap);
    loadBalancer.updateClusterMetrics(clusterStatus);
    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable = (Map) mockClusterServersWithTables(clusterState);
    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
    Set<RegionInfo> regionsMoveFromServerA = new HashSet<>();
    Set<ServerName> targetServers = new HashSet<>();
    for (RegionPlan plan : plans) {
        if (plan.getSource().equals(serverA)) {
            regionsMoveFromServerA.add(plan.getRegionInfo());
            targetServers.add(plan.getDestination());
        }
    }
    // should move 2 regions from serverA, one moves to serverB, the other moves to serverC
    assertEquals(2, regionsMoveFromServerA.size());
    assertEquals(2, targetServers.size());
    assertTrue(regionsOnServerA.containsAll(regionsMoveFromServerA));
}

Example 14 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class TestBalancerDecision method testBalancerDecisions.

@Test
public void testBalancerDecisions() {
    conf.setBoolean("hbase.master.balancer.decision.buffer.enabled", true);
    MasterServices services = mock(MasterServices.class);
    when(services.getConfiguration()).thenReturn(conf);
    MasterClusterInfoProvider provider = new MasterClusterInfoProvider(services);
    loadBalancer.setClusterInfoProvider(provider);
    loadBalancer.onConfigurationChange(conf);
    float minCost = conf.getFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f);
    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f);
    try {
        // Test with/without per table balancer.
        boolean[] perTableBalancerConfigs = { true, false };
        for (boolean isByTable : perTableBalancerConfigs) {
            conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
            loadBalancer.onConfigurationChange(conf);
            for (int[] mockCluster : clusterStateMocks) {
                Map<ServerName, List<RegionInfo>> servers = mockClusterServers(mockCluster);
                Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable = (Map) mockClusterServersWithTables(servers);
                List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
                boolean emptyPlans = plans == null || plans.isEmpty();
                Assert.assertTrue(emptyPlans || needsBalanceIdleRegion(mockCluster));
            }
        }
        final NamedQueueGetRequest namedQueueGetRequest = new NamedQueueGetRequest();
        namedQueueGetRequest.setNamedQueueEvent(BalancerDecisionDetails.BALANCER_DECISION_EVENT);
        namedQueueGetRequest.setBalancerDecisionsRequest(MasterProtos.BalancerDecisionsRequest.getDefaultInstance());
        NamedQueueGetResponse namedQueueGetResponse = provider.getNamedQueueRecorder().getNamedQueueRecords(namedQueueGetRequest);
        List<RecentLogs.BalancerDecision> balancerDecisions = namedQueueGetResponse.getBalancerDecisions();
        MasterProtos.BalancerDecisionsResponse response = MasterProtos.BalancerDecisionsResponse.newBuilder().addAllBalancerDecision(balancerDecisions).build();
        List<LogEntry> balancerDecisionRecords = ProtobufUtil.getBalancerDecisionEntries(response);
        Assert.assertTrue(balancerDecisionRecords.size() > 160);
    } finally {
        // reset config
        conf.unset(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE);
        conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", minCost);
        loadBalancer.onConfigurationChange(conf);
    }
}

Example 15 with RegionPlan

use of org.apache.hadoop.hbase.master.RegionPlan in project hbase by apache.

the class TestReportOnlineRegionsRace method testRace.

@Test
public void testRace() throws Exception {
    RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
    ProcedureExecutor<MasterProcedureEnv> procExec = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
    RegionStateNode rsn = am.getRegionStates().getRegionStateNode(region);
    // halt a regionServerReport
    RESUME_RS_REPORT = new CountDownLatch(1);
    ARRIVE_RS_REPORT = new CountDownLatch(1);
    FINISH_RS_REPORT = new CountDownLatch(1);
    ARRIVE_RS_REPORT.await();
    // schedule a TRSP to REOPEN the region
    RESUME_REPORT_STATE = new CountDownLatch(1);
    Future<byte[]> future = am.moveAsync(new RegionPlan(region, rsn.getRegionLocation(), rsn.getRegionLocation()));
    TransitRegionStateProcedure proc = procExec.getProcedures().stream().filter(p -> p instanceof TransitRegionStateProcedure).filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p).findAny().get();
    IdLock procExecLock = procExec.getProcExecutionLock();
    // a CloseRegionProcedure and then the OpenRegionProcedure we want to block
    IdLock.Entry lockEntry = procExecLock.getLockEntry(proc.getProcId() + 2);
    // resume the reportRegionStateTransition to finish the CloseRegionProcedure
    RESUME_REPORT_STATE.countDown();
    // wait until we schedule the OpenRegionProcedure
    UTIL.waitFor(10000, () -> proc.getCurrentStateId() == REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE);
    // the region should be in OPENING state
    assertEquals(RegionState.State.OPENING, rsn.getState());
    // resume the region server report
    RESUME_RS_REPORT.countDown();
    // wait until it finishes, it will find that the region is opened on the rs
    FINISH_RS_REPORT.await();
    // let the OpenRegionProcedure go
    procExecLock.releaseLockEntry(lockEntry);
    // wait until the TRSP is done
    future.get();
    // change the region state to OPEN
    try (Table table = UTIL.getConnection().getTableBuilder(NAME, null).setWriteRpcTimeout(1000).setOperationTimeout(2000).build()) {
        table.put(new Put(Bytes.toBytes("key")).addColumn(CF, Bytes.toBytes("cq"), Bytes.toBytes("val")));
    }
}