Examples with AssignmentManager org.apache.hadoop.hbase.master.assignment.AssignmentManager used on opensource projects

Example 26 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class TestReopenTableRegionsProcedureBackoff method testRetryBackoff.

@Test
public void testRetryBackoff() throws IOException, InterruptedException {
    AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
    ProcedureExecutor<MasterProcedureEnv> procExec = UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
    RegionInfo regionInfo = UTIL.getAdmin().getRegions(TABLE_NAME).get(0);
    RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(regionInfo);
    // just a dummy one
    TransitRegionStateProcedure trsp = TransitRegionStateProcedure.unassign(procExec.getEnvironment(), regionInfo);
    long openSeqNum;
    regionNode.lock();
    try {
        openSeqNum = regionNode.getOpenSeqNum();
        // make a fake state to let the procedure wait.
        regionNode.setState(State.OPENING);
        regionNode.setOpenSeqNum(-1L);
        regionNode.setProcedure(trsp);
    } finally {
        regionNode.unlock();
    }
    ReopenTableRegionsProcedure proc = new ReopenTableRegionsProcedure(TABLE_NAME);
    procExec.submitProcedure(proc);
    UTIL.waitFor(10000, () -> proc.getState() == ProcedureState.WAITING_TIMEOUT);
    long oldTimeout = 0;
    int timeoutIncrements = 0;
    for (; ; ) {
        long timeout = proc.getTimeout();
        if (timeout > oldTimeout) {
            LOG.info("Timeout incremented, was {}, now is {}, increments={}", timeout, oldTimeout, timeoutIncrements);
            oldTimeout = timeout;
            timeoutIncrements++;
            if (timeoutIncrements > 3) {
                // If we incremented at least twice, break; the backoff is working.
                break;
            }
        }
        Thread.sleep(1000);
    }
    regionNode.lock();
    try {
        // reset to the correct state
        regionNode.setState(State.OPEN);
        regionNode.setOpenSeqNum(openSeqNum);
        regionNode.unsetProcedure(trsp);
    } finally {
        regionNode.unlock();
    }
    ProcedureSyncWait.waitForProcedureToComplete(procExec, proc, 60000);
    assertTrue(regionNode.getOpenSeqNum() > openSeqNum);
}

Example 27 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class TestRegionMergeTransactionOnCluster method testMerge.

/**
 * This test tests 1, merging region not online;
 * 2, merging same two regions; 3, merging unknown regions.
 * They are in one test case so that we don't have to create
 * many tables, and these tests are simple.
 */
@Test
public void testMerge() throws Exception {
    LOG.info("Starting " + name.getMethodName());
    final TableName tableName = TableName.valueOf(name.getMethodName());
    final Admin admin = TEST_UTIL.getAdmin();
    try {
        // Create table and load data.
        Table table = createTableAndLoadData(MASTER, tableName);
        AssignmentManager am = MASTER.getAssignmentManager();
        List<RegionInfo> regions = am.getRegionStates().getRegionsOfTable(tableName);
        // Fake offline one region
        RegionInfo a = regions.get(0);
        RegionInfo b = regions.get(1);
        am.unassign(b);
        am.offlineRegion(b);
        try {
            // Merge offline region. Region a is offline here
            FutureUtils.get(admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false));
            fail("Offline regions should not be able to merge");
        } catch (DoNotRetryRegionException ie) {
            System.out.println(ie);
            assertTrue(ie instanceof MergeRegionException);
        }
        try {
            // Merge the same region: b and b.
            FutureUtils.get(admin.mergeRegionsAsync(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true));
            fail("A region should not be able to merge with itself, even forcfully");
        } catch (IOException ie) {
            assertTrue("Exception should mention regions not online", StringUtils.stringifyException(ie).contains("region to itself") && ie instanceof MergeRegionException);
        }
        try {
            // Merge unknown regions
            FutureUtils.get(admin.mergeRegionsAsync(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true));
            fail("Unknown region could not be merged");
        } catch (IOException ie) {
            assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException);
        }
        table.close();
    } finally {
        TEST_UTIL.deleteTable(tableName);
    }
}

Example 28 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class TestCatalogJanitorInMemoryStates method testInMemoryParentCleanup.

/**
 * Test clearing a split parent from memory.
 */
@Test
public void testInMemoryParentCleanup() throws IOException, InterruptedException, ExecutionException {
    HMaster master = TEST_UTIL.getHBaseCluster().getMaster();
    final AssignmentManager am = master.getAssignmentManager();
    final ServerManager sm = master.getServerManager();
    Admin admin = TEST_UTIL.getAdmin();
    admin.catalogJanitorSwitch(false);
    final TableName tableName = name.getTableName();
    Table t = TEST_UTIL.createTable(tableName, FAMILY);
    TEST_UTIL.loadTable(t, FAMILY, false);
    RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName);
    List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
    // We need to create a valid split with daughter regions
    HRegionLocation parent = allRegionLocations.get(0);
    List<HRegionLocation> daughters = splitRegion(parent.getRegion());
    LOG.info("Parent region: " + parent);
    LOG.info("Daughter regions: " + daughters);
    assertNotNull("Should have found daughter regions for " + parent, daughters);
    assertTrue("Parent region should exist in RegionStates", am.getRegionStates().isRegionInRegionStates(parent.getRegion()));
    assertTrue("Parent region should exist in ServerManager", sm.isRegionInServerManagerStates(parent.getRegion()));
    // clean the parent
    Result r = MetaMockingUtil.getMetaTableRowResult(parent.getRegion(), null, daughters.get(0).getRegion(), daughters.get(1).getRegion());
    CatalogJanitor.cleanParent(master, parent.getRegion(), r);
    // wait for procedures to complete
    Waiter.waitFor(TEST_UTIL.getConfiguration(), 10 * 1000, new Waiter.Predicate<Exception>() {

        @Override
        public boolean evaluate() throws Exception {
            ProcedureExecutor<MasterProcedureEnv> pe = master.getMasterProcedureExecutor();
            for (Procedure<MasterProcedureEnv> proc : pe.getProcedures()) {
                if (proc.getClass().isAssignableFrom(GCRegionProcedure.class) && proc.isFinished()) {
                    return true;
                }
            }
            return false;
        }
    });
    assertFalse("Parent region should have been removed from RegionStates", am.getRegionStates().isRegionInRegionStates(parent.getRegion()));
    assertFalse("Parent region should have been removed from ServerManager", sm.isRegionInServerManagerStates(parent.getRegion()));
}

Example 29 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class TestMetaFixer method testOverlapWithSmallMergeCount.

@Test
public void testOverlapWithSmallMergeCount() throws Exception {
    TableName tn = TableName.valueOf(this.name.getMethodName());
    try {
        testOverlapCommon(tn);
        HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
        CatalogJanitor cj = services.getCatalogJanitor();
        cj.scan();
        Report report = cj.getLastReport();
        assertEquals(6, report.getOverlaps().size());
        assertEquals(2, MetaFixer.calculateMerges(5, report.getOverlaps()).size());
        // The max merge count is set to 5 so overlap regions are divided into
        // two merge requests.
        TEST_UTIL.getHBaseCluster().getMaster().getConfiguration().setInt("hbase.master.metafixer.max.merge.count", 5);
        // Get overlap regions
        HashSet<String> overlapRegions = new HashSet<>();
        for (Pair<RegionInfo, RegionInfo> pair : report.getOverlaps()) {
            overlapRegions.add(pair.getFirst().getRegionNameAsString());
            overlapRegions.add(pair.getSecond().getRegionNameAsString());
        }
        MetaFixer fixer = new MetaFixer(services);
        fixer.fixOverlaps(report);
        AssignmentManager am = services.getAssignmentManager();
        HBaseTestingUtil.await(200, () -> {
            try {
                cj.scan();
                final Report postReport = cj.getLastReport();
                RegionStates regionStates = am.getRegionStates();
                RegionStateStore regionStateStore = am.getRegionStateStore();
                // Make sure that two merged regions are opened and GCs are done.
                if (postReport.getOverlaps().size() == 1) {
                    Pair<RegionInfo, RegionInfo> pair = postReport.getOverlaps().get(0);
                    if ((!overlapRegions.contains(pair.getFirst().getRegionNameAsString()) && regionStates.getRegionState(pair.getFirst()).isOpened()) && (!overlapRegions.contains(pair.getSecond().getRegionNameAsString()) && regionStates.getRegionState(pair.getSecond()).isOpened())) {
                        // Make sure GC is done.
                        List<RegionInfo> firstParents = regionStateStore.getMergeRegions(pair.getFirst());
                        List<RegionInfo> secondParents = regionStateStore.getMergeRegions(pair.getSecond());
                        return (firstParents == null || firstParents.isEmpty()) && (secondParents == null || secondParents.isEmpty());
                    }
                }
                return false;
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Second run of fixOverlap should fix all.
        report = cj.getLastReport();
        fixer.fixOverlaps(report);
        HBaseTestingUtil.await(20, () -> {
            try {
                // Make sure it GC only once.
                return (cj.scan() > 0);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // No holes reported.
        cj.scan();
        final Report postReport = cj.getLastReport();
        assertTrue(postReport.isEmpty());
    } finally {
        TEST_UTIL.getHBaseCluster().getMaster().getConfiguration().unset("hbase.master.metafixer.max.merge.count");
        TEST_UTIL.deleteTable(tn);
    }
}

Example 30 with AssignmentManager

use of org.apache.hadoop.hbase.master.assignment.AssignmentManager in project hbase by apache.

the class RSGroupableBalancerTestBase method getMockedMaster.

protected static MasterServices getMockedMaster() throws IOException {
    TableDescriptors tds = Mockito.mock(TableDescriptors.class);
    Mockito.when(tds.get(tables[0])).thenReturn(tableDescs.get(tables[0]));
    Mockito.when(tds.get(tables[1])).thenReturn(tableDescs.get(tables[1]));
    Mockito.when(tds.get(tables[2])).thenReturn(tableDescs.get(tables[2]));
    Mockito.when(tds.get(tables[3])).thenReturn(tableDescs.get(tables[3]));
    MasterServices services = Mockito.mock(HMaster.class);
    Mockito.when(services.getTableDescriptors()).thenReturn(tds);
    AssignmentManager am = Mockito.mock(AssignmentManager.class);
    Mockito.when(services.getAssignmentManager()).thenReturn(am);
    Mockito.when(services.getConfiguration()).thenReturn(conf);
    RSGroupInfoManager manager = getMockedGroupInfoManager();
    Mockito.when(services.getRSGroupInfoManager()).thenReturn(manager);
    return services;
}