Examples with RegionStateStore org.apache.hadoop.hbase.master.assignment.RegionStateStore used on opensource projects

Example 1 with RegionStateStore

use of org.apache.hadoop.hbase.master.assignment.RegionStateStore in project hbase by apache.

the class RestoreSnapshotProcedure method updateMETA.

/**
 * Apply changes to hbase:meta
 */
private void updateMETA(final MasterProcedureEnv env) throws IOException {
    try {
        Connection conn = env.getMasterServices().getConnection();
        RegionStateStore regionStateStore = env.getAssignmentManager().getRegionStateStore();
        int regionReplication = modifiedTableDescriptor.getRegionReplication();
        // 1. Prepare to restore
        getMonitorStatus().setStatus("Preparing to restore each region");
        // that are not correct after the restore.
        if (regionsToRemove != null) {
            regionStateStore.deleteRegions(regionsToRemove);
            deleteRegionsFromInMemoryStates(regionsToRemove, env, regionReplication);
        }
        // in the snapshot folder.
        if (regionsToAdd != null) {
            MetaTableAccessor.addRegionsToMeta(conn, regionsToAdd, regionReplication);
            addRegionsToInMemoryStates(regionsToAdd, env, regionReplication);
        }
        if (regionsToRestore != null) {
            regionStateStore.overwriteRegions(regionsToRestore, regionReplication);
            deleteRegionsFromInMemoryStates(regionsToRestore, env, regionReplication);
            addRegionsToInMemoryStates(regionsToRestore, env, regionReplication);
        }
        RestoreSnapshotHelper.RestoreMetaChanges metaChanges = new RestoreSnapshotHelper.RestoreMetaChanges(modifiedTableDescriptor, parentsToChildrenPairMap);
        metaChanges.updateMetaParentRegions(conn, regionsToAdd);
        // At this point the restore is complete.
        LOG.info("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " on table=" + getTableName() + " completed!");
    } catch (IOException e) {
        final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();
        String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) + " failed in meta update. Try re-running the restore command.";
        LOG.error(msg, e);
        monitorException.receive(new ForeignException(env.getMasterServices().getServerName().toString(), e));
        throw new IOException(msg, e);
    }
    monitorStatus.markComplete("Restore snapshot '" + snapshot.getName() + "'!");
    MetricsSnapshot metricsSnapshot = new MetricsSnapshot();
    metricsSnapshot.addSnapshotRestore(monitorStatus.getCompletionTimestamp() - monitorStatus.getStartTime());
}

Example 2 with RegionStateStore

use of org.apache.hadoop.hbase.master.assignment.RegionStateStore in project hbase by apache.

the class TestCatalogJanitorCluster method verifyCornerHoles.

private void verifyCornerHoles(CatalogJanitor janitor, TableName tableName) throws IOException {
    RegionStateStore regionStateStore = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStateStore();
    RegionInfo firstRegion = getRegionInfo(tableName, "".getBytes());
    RegionInfo secondRegion = getRegionInfo(tableName, "bbb".getBytes());
    regionStateStore.deleteRegion(firstRegion);
    LinkedList<Pair<RegionInfo, RegionInfo>> holes = getHoles(janitor, tableName);
    assertEquals(1, holes.size());
    Pair<RegionInfo, RegionInfo> regionInfoRegionInfoPair = holes.get(0);
    assertTrue(regionInfoRegionInfoPair.getFirst().getTable().equals(RegionInfoBuilder.UNDEFINED.getTable()));
    assertTrue(regionInfoRegionInfoPair.getSecond().getTable().equals(tableName));
    assertTrue(regionInfoRegionInfoPair.getSecond().getEncodedName().equals(secondRegion.getEncodedName()));
    RegionInfo lastRegion = getRegionInfo(tableName, "zzz".getBytes());
    RegionInfo secondLastRegion = getRegionInfo(tableName, "yyy".getBytes());
    regionStateStore.deleteRegion(lastRegion);
    holes = getHoles(janitor, tableName);
    assertEquals(2, holes.size());
    regionInfoRegionInfoPair = holes.get(1);
    assertTrue(regionInfoRegionInfoPair.getFirst().getEncodedName().equals(secondLastRegion.getEncodedName()));
    assertTrue(regionInfoRegionInfoPair.getSecond().getTable().equals(RegionInfoBuilder.UNDEFINED.getTable()));
}

Example 3 with RegionStateStore

use of org.apache.hadoop.hbase.master.assignment.RegionStateStore in project hbase by apache.

the class TestCatalogJanitorCluster method testConsistency.

/**
 * Fat method where we start with a fat hbase:meta and then gradually intro problems running
 * catalogjanitor for each to ensure it triggers complaint. Do one big method because takes a
 * while to build up the context we need. We create three tables and then make holes, overlaps,
 * add unknown servers and empty out regioninfo columns. Each should up counts in the
 * CatalogJanitor.Report produced.
 */
@Test
public void testConsistency() throws IOException {
    CatalogJanitor janitor = TEST_UTIL.getHBaseCluster().getMaster().getCatalogJanitor();
    RegionStateStore regionStateStore = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStateStore();
    janitor.scan();
    Report report = janitor.getLastReport();
    // Assert no problems.
    assertTrue(report.isEmpty());
    // Now remove first region in table t2 to see if catalogjanitor scan notices.
    List<RegionInfo> t2Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T2);
    regionStateStore.deleteRegion(t2Ris.get(0));
    janitor.scan();
    report = janitor.getLastReport();
    assertFalse(report.isEmpty());
    assertEquals(1, report.getHoles().size());
    assertTrue(report.getHoles().get(0).getFirst().getTable().equals(RegionInfoBuilder.UNDEFINED.getTable()));
    assertTrue(report.getHoles().get(0).getSecond().getTable().equals(T2));
    assertEquals(0, report.getOverlaps().size());
    // Next, add overlaps to first row in t3
    List<RegionInfo> t3Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T3);
    RegionInfo ri = t3Ris.get(0);
    RegionInfo newRi1 = RegionInfoBuilder.newBuilder(ri.getTable()).setStartKey(incrementRow(ri.getStartKey())).setEndKey(incrementRow(ri.getEndKey())).build();
    Put p1 = MetaTableAccessor.makePutFromRegionInfo(newRi1, EnvironmentEdgeManager.currentTime());
    RegionInfo newRi2 = RegionInfoBuilder.newBuilder(newRi1.getTable()).setStartKey(incrementRow(newRi1.getStartKey())).setEndKey(incrementRow(newRi1.getEndKey())).build();
    Put p2 = MetaTableAccessor.makePutFromRegionInfo(newRi2, EnvironmentEdgeManager.currentTime());
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(p1, p2));
    janitor.scan();
    report = janitor.getLastReport();
    assertFalse(report.isEmpty());
    // We added two overlaps so total three.
    assertEquals(3, report.getOverlaps().size());
    // Assert hole is still there.
    assertEquals(1, report.getHoles().size());
    // Assert other attributes are empty still.
    assertTrue(report.getEmptyRegionInfo().isEmpty());
    assertTrue(report.getUnknownServers().isEmpty());
    // Now make bad server in t1.
    List<RegionInfo> t1Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T1);
    RegionInfo t1Ri1 = t1Ris.get(1);
    Put pServer = new Put(t1Ri1.getRegionName());
    pServer.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(0), Bytes.toBytes("bad.server.example.org:1234"));
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pServer));
    janitor.scan();
    report = janitor.getLastReport();
    assertFalse(report.isEmpty());
    assertEquals(1, report.getUnknownServers().size());
    // Test what happens if we blow away an info:server row, if it is null. Should not kill CJ
    // and we should log the row that had the problem. HBASE-23192. Just make sure we don't
    // break if this happens.
    LOG.info("Make null info:server");
    Put emptyInfoServerPut = new Put(t1Ri1.getRegionName());
    emptyInfoServerPut.addColumn(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(0), Bytes.toBytes(""));
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(emptyInfoServerPut));
    janitor.scan();
    report = janitor.getLastReport();
    assertEquals(0, report.getUnknownServers().size());
    // Mke an empty regioninfo in t1.
    RegionInfo t1Ri2 = t1Ris.get(2);
    Put pEmptyRI = new Put(t1Ri2.getRegionName());
    pEmptyRI.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY);
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(pEmptyRI));
    janitor.scan();
    report = janitor.getLastReport();
    assertEquals(1, report.getEmptyRegionInfo().size());
    int holesReported = report.getHoles().size();
    int overlapsReported = report.getOverlaps().size();
    // Test the case for T4
    // r1: [aa, bb), r2: [cc, dd), r3: [a, cc)
    // Make sure only overlaps and no holes are reported.
    List<RegionInfo> t4Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T4);
    // delete the region [bb, cc)
    regionStateStore.deleteRegion(t4Ris.get(2));
    // add a new region [a, cc)
    RegionInfo newRiT4 = RegionInfoBuilder.newBuilder(T4).setStartKey("a".getBytes()).setEndKey("cc".getBytes()).build();
    Put putForT4 = MetaTableAccessor.makePutFromRegionInfo(newRiT4, EnvironmentEdgeManager.currentTime());
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT4));
    janitor.scan();
    report = janitor.getLastReport();
    // there is no new hole reported, 2 more overLaps added.
    assertEquals(holesReported, report.getHoles().size());
    assertEquals(overlapsReported + 2, report.getOverlaps().size());
    holesReported = report.getHoles().size();
    overlapsReported = report.getOverlaps().size();
    // Test the case for T5
    // r0: [, bb), r1: [a, g), r2: [bb, cc), r3: [dd, )
    // Make sure only overlaps and no holes are reported.
    List<RegionInfo> t5Ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), T5);
    // delete the region [cc, dd)
    regionStateStore.deleteRegion(t5Ris.get(2));
    // add a new region [a, g)
    RegionInfo newRiT5 = RegionInfoBuilder.newBuilder(T5).setStartKey("a".getBytes()).setEndKey("g".getBytes()).build();
    Put putForT5 = MetaTableAccessor.makePutFromRegionInfo(newRiT5, EnvironmentEdgeManager.currentTime());
    MetaTableAccessor.putsToMetaTable(TEST_UTIL.getConnection(), Arrays.asList(putForT5));
    janitor.scan();
    report = janitor.getLastReport();
    // there is no new hole reported, 3 more overLaps added.
    // ([a, g), [, bb)), ([a, g), [bb, cc)), ([a, g), [dd, ))
    assertEquals(holesReported, report.getHoles().size());
    assertEquals(overlapsReported + 3, report.getOverlaps().size());
}

Example 4 with RegionStateStore

use of org.apache.hadoop.hbase.master.assignment.RegionStateStore in project hbase by apache.

the class TestMetaFixer method testOverlapWithSmallMergeCount.

@Test
public void testOverlapWithSmallMergeCount() throws Exception {
    TableName tn = TableName.valueOf(this.name.getMethodName());
    try {
        testOverlapCommon(tn);
        HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
        CatalogJanitor cj = services.getCatalogJanitor();
        cj.scan();
        Report report = cj.getLastReport();
        assertEquals(6, report.getOverlaps().size());
        assertEquals(2, MetaFixer.calculateMerges(5, report.getOverlaps()).size());
        // The max merge count is set to 5 so overlap regions are divided into
        // two merge requests.
        TEST_UTIL.getHBaseCluster().getMaster().getConfiguration().setInt("hbase.master.metafixer.max.merge.count", 5);
        // Get overlap regions
        HashSet<String> overlapRegions = new HashSet<>();
        for (Pair<RegionInfo, RegionInfo> pair : report.getOverlaps()) {
            overlapRegions.add(pair.getFirst().getRegionNameAsString());
            overlapRegions.add(pair.getSecond().getRegionNameAsString());
        }
        MetaFixer fixer = new MetaFixer(services);
        fixer.fixOverlaps(report);
        AssignmentManager am = services.getAssignmentManager();
        HBaseTestingUtil.await(200, () -> {
            try {
                cj.scan();
                final Report postReport = cj.getLastReport();
                RegionStates regionStates = am.getRegionStates();
                RegionStateStore regionStateStore = am.getRegionStateStore();
                // Make sure that two merged regions are opened and GCs are done.
                if (postReport.getOverlaps().size() == 1) {
                    Pair<RegionInfo, RegionInfo> pair = postReport.getOverlaps().get(0);
                    if ((!overlapRegions.contains(pair.getFirst().getRegionNameAsString()) && regionStates.getRegionState(pair.getFirst()).isOpened()) && (!overlapRegions.contains(pair.getSecond().getRegionNameAsString()) && regionStates.getRegionState(pair.getSecond()).isOpened())) {
                        // Make sure GC is done.
                        List<RegionInfo> firstParents = regionStateStore.getMergeRegions(pair.getFirst());
                        List<RegionInfo> secondParents = regionStateStore.getMergeRegions(pair.getSecond());
                        return (firstParents == null || firstParents.isEmpty()) && (secondParents == null || secondParents.isEmpty());
                    }
                }
                return false;
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Second run of fixOverlap should fix all.
        report = cj.getLastReport();
        fixer.fixOverlaps(report);
        HBaseTestingUtil.await(20, () -> {
            try {
                // Make sure it GC only once.
                return (cj.scan() > 0);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // No holes reported.
        cj.scan();
        final Report postReport = cj.getLastReport();
        assertTrue(postReport.isEmpty());
    } finally {
        TEST_UTIL.getHBaseCluster().getMaster().getConfiguration().unset("hbase.master.metafixer.max.merge.count");
        TEST_UTIL.deleteTable(tn);
    }
}