Examples with AssignmentManager org.apache.hadoop.hbase.master.assignment.AssignmentManager used on opensource projects

Example 16 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class TestMasterBalancerNPE method testBalancerNPE.

/**
 * This test is for HBASE-26712, to make the region is unassigned just before
 * {@link AssignmentManager#balance} is invoked on the region.
 */
@Test
public void testBalancerNPE() throws Exception {
    TEST_UTIL.startMiniCluster(2);
    TEST_UTIL.getAdmin().balancerSwitch(false, true);
    TableName tableName = createTable(name.getMethodName());
    final HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
    List<RegionInfo> regionInfos = TEST_UTIL.getAdmin().getRegions(tableName);
    assertTrue(regionInfos.size() == 1);
    final ServerName serverName1 = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
    final ServerName serverName2 = TEST_UTIL.getMiniHBaseCluster().getRegionServer(1).getServerName();
    final RegionInfo regionInfo = regionInfos.get(0);
    RSGroupBasedLoadBalancer loadBalancer = master.getLoadBalancer();
    RSGroupBasedLoadBalancer spiedLoadBalancer = Mockito.spy(loadBalancer);
    final AtomicReference<RegionPlan> regionPlanRef = new AtomicReference<RegionPlan>();
    /**
     * Mock {@link RSGroupBasedLoadBalancer#balanceCluster} to return the {@link RegionPlan} to move
     * the only region to the other RegionServer.
     */
    Mockito.doAnswer((InvocationOnMock invocation) -> {
        @SuppressWarnings("unchecked") Map<TableName, Map<ServerName, List<RegionInfo>>> tableNameToRegionServerNameToRegionInfos = (Map<TableName, Map<ServerName, List<RegionInfo>>>) invocation.getArgument(0);
        Map<ServerName, List<RegionInfo>> regionServerNameToRegionInfos = tableNameToRegionServerNameToRegionInfos.get(tableName);
        assertTrue(regionServerNameToRegionInfos.size() == 2);
        List<ServerName> assignedRegionServerNames = new ArrayList<ServerName>();
        for (Map.Entry<ServerName, List<RegionInfo>> entry : regionServerNameToRegionInfos.entrySet()) {
            if (entry.getValue().size() > 0) {
                assignedRegionServerNames.add(entry.getKey());
            }
        }
        assertTrue(assignedRegionServerNames.size() == 1);
        ServerName assignedRegionServerName = assignedRegionServerNames.get(0);
        ServerName notAssignedRegionServerName = assignedRegionServerName.equals(serverName1) ? serverName2 : serverName1;
        RegionPlan regionPlan = new RegionPlan(regionInfo, assignedRegionServerName, notAssignedRegionServerName);
        regionPlanRef.set(regionPlan);
        return Arrays.asList(regionPlan);
    }).when(spiedLoadBalancer).balanceCluster(Mockito.anyMap());
    AssignmentManager assignmentManager = master.getAssignmentManager();
    final AssignmentManager spiedAssignmentManager = Mockito.spy(assignmentManager);
    final CyclicBarrier cyclicBarrier = new CyclicBarrier(2);
    /**
     * Override {@link AssignmentManager#balance} to invoke real {@link AssignmentManager#balance}
     * after the region is successfully unassigned.
     */
    Mockito.doAnswer((InvocationOnMock invocation) -> {
        RegionPlan regionPlan = invocation.getArgument(0);
        RegionPlan referedRegionPlan = regionPlanRef.get();
        assertTrue(referedRegionPlan != null);
        if (referedRegionPlan.equals(regionPlan)) {
            /**
             * To make {@link AssignmentManager#unassign} could be invoked just before
             * {@link AssignmentManager#balance} is invoked.
             */
            cyclicBarrier.await();
            /**
             * After {@link AssignmentManager#unassign} is completed,we could invoke
             * {@link AssignmentManager#balance}.
             */
            cyclicBarrier.await();
        }
        /**
         * Before HBASE-26712,here may throw NPE.
         */
        return invocation.callRealMethod();
    }).when(spiedAssignmentManager).balance(Mockito.any());
    try {
        final AtomicReference<Throwable> exceptionRef = new AtomicReference<Throwable>(null);
        Thread unassignThread = new Thread(() -> {
            try {
                /**
                 * To invoke {@link AssignmentManager#unassign} just before
                 * {@link AssignmentManager#balance} is invoked.
                 */
                cyclicBarrier.await();
                spiedAssignmentManager.unassign(regionInfo);
                assertTrue(spiedAssignmentManager.getRegionStates().getRegionAssignments().get(regionInfo) == null);
                /**
                 * After {@link AssignmentManager#unassign} is completed,we could invoke
                 * {@link AssignmentManager#balance}.
                 */
                cyclicBarrier.await();
            } catch (Exception e) {
                exceptionRef.set(e);
            }
        });
        unassignThread.setName("UnassignThread");
        unassignThread.start();
        master.setLoadBalancer(spiedLoadBalancer);
        master.setAssignmentManager(spiedAssignmentManager);
        /**
         * enable balance
         */
        TEST_UTIL.getAdmin().balancerSwitch(true, false);
        /**
         * Before HBASE-26712,here invokes {@link AssignmentManager#balance(RegionPlan)}
         * which may throw NPE.
         */
        master.balanceOrUpdateMetrics();
        unassignThread.join();
        assertTrue(exceptionRef.get() == null);
    } finally {
        master.setLoadBalancer(loadBalancer);
        master.setAssignmentManager(assignmentManager);
    }
}

Example 17 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class HBaseTestingUtility method predicateNoRegionsInTransition.

/**
 * Returns a {@link Predicate} for checking that there are no regions in transition in master
 */
public ExplainingPredicate<IOException> predicateNoRegionsInTransition() {
    return new ExplainingPredicate<IOException>() {

        @Override
        public String explainFailure() throws IOException {
            final RegionStates regionStates = getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
            return "found in transition: " + regionStates.getRegionsInTransition().toString();
        }

        @Override
        public boolean evaluate() throws IOException {
            HMaster master = getMiniHBaseCluster().getMaster();
            if (master == null)
                return false;
            AssignmentManager am = master.getAssignmentManager();
            if (am == null)
                return false;
            return !am.hasRegionsInTransition();
        }
    };
}

Example 18 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class HBaseTestingUtility method assignRegion.

/**
 * Uses directly the assignment manager to assign the region. and waits until the specified region
 * has completed assignment.
 * @return true if the region is assigned false otherwise.
 */
public boolean assignRegion(final RegionInfo regionInfo) throws IOException, InterruptedException {
    final AssignmentManager am = getHBaseCluster().getMaster().getAssignmentManager();
    am.assign(regionInfo);
    return AssignmentTestingUtil.waitForAssignment(am, regionInfo);
}

Example 19 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class TestHbck method testSetRegionStateInMeta.

@Test
public void testSetRegionStateInMeta() throws Exception {
    Hbck hbck = getHbck();
    Admin admin = TEST_UTIL.getAdmin();
    final List<RegionInfo> regions = admin.getRegions(TABLE_NAME);
    final AssignmentManager am = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
    Map<String, RegionState.State> prevStates = new HashMap<>();
    Map<String, RegionState.State> newStates = new HashMap<>();
    final Map<String, Pair<RegionState.State, RegionState.State>> regionsMap = new HashMap<>();
    regions.forEach(r -> {
        RegionState prevState = am.getRegionStates().getRegionState(r);
        prevStates.put(r.getEncodedName(), prevState.getState());
        newStates.put(r.getEncodedName(), RegionState.State.CLOSED);
        regionsMap.put(r.getEncodedName(), new Pair<>(prevState.getState(), RegionState.State.CLOSED));
    });
    final Map<String, RegionState.State> result = hbck.setRegionStateInMeta(newStates);
    result.forEach((k, v) -> {
        RegionState.State prevState = regionsMap.get(k).getFirst();
        assertEquals(prevState, v);
    });
    regions.forEach(r -> {
        RegionState cachedState = am.getRegionStates().getRegionState(r.getEncodedName());
        RegionState.State newState = regionsMap.get(r.getEncodedName()).getSecond();
        assertEquals(newState, cachedState.getState());
    });
    hbck.setRegionStateInMeta(prevStates);
}

Example 20 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class ServerCrashProcedure method executeFromState.

@Override
protected Flow executeFromState(MasterProcedureEnv env, ServerCrashState state) throws ProcedureSuspendedException, ProcedureYieldException {
    final MasterServices services = env.getMasterServices();
    final AssignmentManager am = env.getAssignmentManager();
    updateProgress(true);
    // Server gets removed from processing list below on procedure successful finish.
    if (!notifiedDeadServer) {
        notifiedDeadServer = true;
    }
    switch(state) {
        case SERVER_CRASH_START:
        case SERVER_CRASH_SPLIT_META_LOGS:
        case SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR:
        case SERVER_CRASH_ASSIGN_META:
            break;
        default:
            // If hbase:meta is not assigned, yield.
            if (env.getAssignmentManager().waitMetaLoaded(this)) {
                throw new ProcedureSuspendedException();
            }
    }
    try {
        switch(state) {
            case SERVER_CRASH_START:
                LOG.info("Start " + this);
                // If carrying meta, process it first. Else, get list of regions on crashed server.
                if (this.carryingMeta) {
                    setNextState(ServerCrashState.SERVER_CRASH_SPLIT_META_LOGS);
                } else {
                    setNextState(ServerCrashState.SERVER_CRASH_GET_REGIONS);
                }
                break;
            case SERVER_CRASH_SPLIT_META_LOGS:
                if (env.getMasterConfiguration().getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK, DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)) {
                    zkCoordinatedSplitMetaLogs(env);
                    setNextState(ServerCrashState.SERVER_CRASH_ASSIGN_META);
                } else {
                    am.getRegionStates().metaLogSplitting(serverName);
                    addChildProcedure(createSplittingWalProcedures(env, true));
                    setNextState(ServerCrashState.SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR);
                }
                break;
            case SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR:
                if (isSplittingDone(env, true)) {
                    setNextState(ServerCrashState.SERVER_CRASH_ASSIGN_META);
                    am.getRegionStates().metaLogSplit(serverName);
                } else {
                    setNextState(ServerCrashState.SERVER_CRASH_SPLIT_META_LOGS);
                }
                break;
            case SERVER_CRASH_ASSIGN_META:
                assignRegions(env, Arrays.asList(RegionInfoBuilder.FIRST_META_REGIONINFO));
                setNextState(ServerCrashState.SERVER_CRASH_GET_REGIONS);
                break;
            case SERVER_CRASH_GET_REGIONS:
                this.regionsOnCrashedServer = getRegionsOnCrashedServer(env);
                // if we should do distributed log splitting.
                if (regionsOnCrashedServer != null) {
                    LOG.info("{} had {} regions", serverName, regionsOnCrashedServer.size());
                    if (LOG.isTraceEnabled()) {
                        this.regionsOnCrashedServer.stream().forEach(ri -> LOG.trace(ri.getShortNameToLog()));
                    }
                }
                if (!this.shouldSplitWal) {
                    setNextState(ServerCrashState.SERVER_CRASH_ASSIGN);
                } else {
                    setNextState(ServerCrashState.SERVER_CRASH_SPLIT_LOGS);
                }
                break;
            case SERVER_CRASH_SPLIT_LOGS:
                if (env.getMasterConfiguration().getBoolean(HBASE_SPLIT_WAL_COORDINATED_BY_ZK, DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)) {
                    zkCoordinatedSplitLogs(env);
                    setNextState(ServerCrashState.SERVER_CRASH_ASSIGN);
                } else {
                    am.getRegionStates().logSplitting(this.serverName);
                    addChildProcedure(createSplittingWalProcedures(env, false));
                    setNextState(ServerCrashState.SERVER_CRASH_DELETE_SPLIT_WALS_DIR);
                }
                break;
            case SERVER_CRASH_DELETE_SPLIT_WALS_DIR:
                if (isSplittingDone(env, false)) {
                    cleanupSplitDir(env);
                    setNextState(ServerCrashState.SERVER_CRASH_ASSIGN);
                    am.getRegionStates().logSplit(this.serverName);
                } else {
                    setNextState(ServerCrashState.SERVER_CRASH_SPLIT_LOGS);
                }
                break;
            case SERVER_CRASH_ASSIGN:
                // Filter changes this.regionsOnCrashedServer.
                if (filterDefaultMetaRegions()) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Assigning regions " + RegionInfo.getShortNameToLog(regionsOnCrashedServer) + ", " + this + "; cycles=" + getCycles());
                    }
                    assignRegions(env, regionsOnCrashedServer);
                }
                setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
                break;
            case SERVER_CRASH_HANDLE_RIT2:
                // Noop. Left in place because we used to call handleRIT here for a second time
                // but no longer necessary since HBASE-20634.
                setNextState(ServerCrashState.SERVER_CRASH_CLAIM_REPLICATION_QUEUES);
                break;
            case SERVER_CRASH_CLAIM_REPLICATION_QUEUES:
                addChildProcedure(new ClaimReplicationQueuesProcedure(serverName));
                setNextState(ServerCrashState.SERVER_CRASH_FINISH);
                break;
            case SERVER_CRASH_FINISH:
                LOG.info("removed crashed server {} after splitting done", serverName);
                services.getAssignmentManager().getRegionStates().removeServer(serverName);
                updateProgress(true);
                return Flow.NO_MORE_STATE;
            default:
                throw new UnsupportedOperationException("unhandled state=" + state);
        }
    } catch (IOException e) {
        LOG.warn("Failed state=" + state + ", retry " + this + "; cycles=" + getCycles(), e);
    }
    return Flow.HAS_MORE_STATE;
}