Examples with HRegionServer org.apache.hadoop.hbase.regionserver.HRegionServer used on opensource projects

Example 76 with HRegionServer

use of org.apache.hadoop.hbase.regionserver.HRegionServer in project hbase by apache.

the class TestTableFavoredNodes method checkIfFavoredNodeInformationIsCorrect.

/*
   * This checks the following:
   *
   * 1. Do all regions of the table have favored nodes updated in master?
   * 2. Is the number of favored nodes correct for a region? Is the start code -1?
   * 3. Is the FN information consistent between Master and the respective RegionServer?
   */
private void checkIfFavoredNodeInformationIsCorrect(TableName tableName) throws Exception {
    /*
     * Since we need HRegionServer to check for consistency of FN between Master and RS,
     * lets construct a map for each serverName lookup. Makes it easy later.
     */
    Map<ServerName, HRegionServer> snRSMap = Maps.newHashMap();
    for (JVMClusterUtil.RegionServerThread rst : TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()) {
        snRSMap.put(rst.getRegionServer().getServerName(), rst.getRegionServer());
    }
    int dnPort = FavoredNodeAssignmentHelper.getDataNodePort(TEST_UTIL.getConfiguration());
    RegionLocator regionLocator = admin.getConnection().getRegionLocator(tableName);
    for (HRegionLocation regionLocation : regionLocator.getAllRegionLocations()) {
        RegionInfo regionInfo = regionLocation.getRegion();
        List<ServerName> fnList = fnm.getFavoredNodes(regionInfo);
        // 1. Does each region have favored node?
        assertNotNull("Favored nodes should not be null for region:" + regionInfo, fnList);
        // 2. Do we have the right number of favored nodes? Is start code -1?
        assertEquals("Incorrect favored nodes for region:" + regionInfo + " fnlist: " + fnList, FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, fnList.size());
        for (ServerName sn : fnList) {
            assertEquals("FN should not have startCode, fnlist:" + fnList, -1, sn.getStartcode());
        }
        // 3. Check if the regionServers have all the FN updated and in sync with Master
        HRegionServer regionServer = snRSMap.get(regionLocation.getServerName());
        assertNotNull("RS should not be null for regionLocation: " + regionLocation, regionServer);
        InetSocketAddress[] rsFavNodes = regionServer.getFavoredNodesForRegion(regionInfo.getEncodedName());
        assertNotNull("RS " + regionLocation.getServerName() + " does not have FN for region: " + regionInfo, rsFavNodes);
        assertEquals("Incorrect FN for region:" + regionInfo.getEncodedName() + " on server:" + regionLocation.getServerName(), FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, rsFavNodes.length);
        // 4. Does DN port match all FN node list?
        for (ServerName sn : fnm.getFavoredNodesWithDNPort(regionInfo)) {
            assertEquals("FN should not have startCode, fnlist:" + fnList, -1, sn.getStartcode());
            assertEquals("FN port should belong to DN port, fnlist:" + fnList, dnPort, sn.getPort());
        }
    }
}

Example 77 with HRegionServer

use of org.apache.hadoop.hbase.regionserver.HRegionServer in project hbase by apache.

the class TestTableSnapshotScanner method testMergeRegion.

@Test
public void testMergeRegion() throws Exception {
    setupCluster();
    TableName tableName = TableName.valueOf("testMergeRegion");
    String snapshotName = tableName.getNameAsString() + "_snapshot";
    Configuration conf = UTIL.getConfiguration();
    Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
    // 20s
    long timeout = 20000;
    try (Admin admin = UTIL.getAdmin()) {
        List<String> serverList = admin.getRegionServers().stream().map(sn -> sn.getServerName()).collect(Collectors.toList());
        // create table with 3 regions
        Table table = UTIL.createTable(tableName, FAMILIES, 1, bbb, yyy, 3);
        List<RegionInfo> regions = admin.getRegions(tableName);
        Assert.assertEquals(3, regions.size());
        RegionInfo region0 = regions.get(0);
        RegionInfo region1 = regions.get(1);
        RegionInfo region2 = regions.get(2);
        // put some data in the table
        UTIL.loadTable(table, FAMILIES);
        admin.flush(tableName);
        // wait flush is finished
        UTIL.waitFor(timeout, () -> {
            try {
                Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
                for (RegionInfo region : regions) {
                    Path regionDir = new Path(tableDir, region.getEncodedName());
                    for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
                        if (fs.listStatus(familyDir).length != 1) {
                            return false;
                        }
                    }
                }
                return true;
            } catch (IOException e) {
                LOG.warn("Failed check if flush is finished", e);
                return false;
            }
        });
        // merge 2 regions
        admin.compactionSwitch(false, serverList);
        admin.mergeRegionsAsync(region0.getEncodedNameAsBytes(), region1.getEncodedNameAsBytes(), true);
        UTIL.waitFor(timeout, () -> admin.getRegions(tableName).size() == 2);
        List<RegionInfo> mergedRegions = admin.getRegions(tableName);
        RegionInfo mergedRegion = mergedRegions.get(0).getEncodedName().equals(region2.getEncodedName()) ? mergedRegions.get(1) : mergedRegions.get(0);
        // snapshot
        admin.snapshot(snapshotName, tableName);
        Assert.assertEquals(1, admin.listSnapshots().size());
        // major compact
        admin.compactionSwitch(true, serverList);
        admin.majorCompactRegion(mergedRegion.getRegionName());
        // wait until merged region has no reference
        UTIL.waitFor(timeout, () -> {
            try {
                for (RegionServerThread regionServerThread : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
                    HRegionServer regionServer = regionServerThread.getRegionServer();
                    for (HRegion subRegion : regionServer.getRegions(tableName)) {
                        if (subRegion.getRegionInfo().getEncodedName().equals(mergedRegion.getEncodedName())) {
                            regionServer.getCompactedHFilesDischarger().chore();
                        }
                    }
                }
                Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
                HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(UTIL.getConfiguration(), fs, tableDir, mergedRegion, true);
                return !regionFs.hasReferences(admin.getDescriptor(tableName));
            } catch (IOException e) {
                LOG.warn("Failed check merged region has no reference", e);
                return false;
            }
        });
        // run catalog janitor to clean and wait for parent regions are archived
        UTIL.getMiniHBaseCluster().getMaster().getCatalogJanitor().choreForTesting();
        UTIL.waitFor(timeout, () -> {
            try {
                Path tableDir = CommonFSUtils.getTableDir(rootDir, tableName);
                for (FileStatus fileStatus : fs.listStatus(tableDir)) {
                    String name = fileStatus.getPath().getName();
                    if (name.equals(region0.getEncodedName()) || name.equals(region1.getEncodedName())) {
                        return false;
                    }
                }
                return true;
            } catch (IOException e) {
                LOG.warn("Check if parent regions are archived error", e);
                return false;
            }
        });
        // set file modify time and then run cleaner
        long time = EnvironmentEdgeManager.currentTime() - TimeToLiveHFileCleaner.DEFAULT_TTL * 1000;
        traverseAndSetFileTime(HFileArchiveUtil.getArchivePath(conf), time);
        UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().runCleaner();
        // scan snapshot
        try (TableSnapshotScanner scanner = new TableSnapshotScanner(conf, UTIL.getDataTestDirOnTestFS(snapshotName), snapshotName, new Scan().withStartRow(bbb).withStopRow(yyy))) {
            verifyScanner(scanner, bbb, yyy);
        }
    } catch (Exception e) {
        LOG.error("scan snapshot error", e);
        Assert.fail("Should not throw Exception: " + e.getMessage());
    } finally {
        tearDownCluster();
    }
}

Example 78 with HRegionServer

use of org.apache.hadoop.hbase.regionserver.HRegionServer in project hbase by apache.

the class TestCoprocessorMetrics method testRegionObserverAfterRegionClosed.

@Test
public void testRegionObserverAfterRegionClosed() throws IOException {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    try (Connection connection = ConnectionFactory.createConnection(UTIL.getConfiguration());
        Admin admin = connection.getAdmin()) {
        admin.createTable(TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder.of(foo)).setCoprocessor(CustomRegionObserver.class.getName()).build(), new byte[][] { foo });
        // create with 2 regions
        try (Table table = connection.getTable(tableName)) {
            table.get(new Get(foo));
            // 2 gets
            table.get(new Get(foo));
        }
        assertPreGetRequestsCounter(CustomRegionObserver.class);
        // close one of the regions
        try (RegionLocator locator = connection.getRegionLocator(tableName)) {
            HRegionLocation loc = locator.getRegionLocation(foo);
            admin.unassign(loc.getRegion().getEncodedNameAsBytes(), true);
            HRegionServer server = UTIL.getMiniHBaseCluster().getRegionServer(loc.getServerName());
            UTIL.waitFor(30000, () -> server.getOnlineRegion(loc.getRegion().getRegionName()) == null);
            assertNull(server.getOnlineRegion(loc.getRegion().getRegionName()));
        }
        // with only 1 region remaining, we should still be able to find the Counter
        assertPreGetRequestsCounter(CustomRegionObserver.class);
        // close the table
        admin.disableTable(tableName);
        MetricRegistryInfo info = MetricsCoprocessor.createRegistryInfoForRegionCoprocessor(CustomRegionObserver.class.getName());
        // ensure that MetricRegistry is deleted
        Optional<MetricRegistry> registry = MetricRegistries.global().get(info);
        assertFalse(registry.isPresent());
    }
}

Example 79 with HRegionServer

use of org.apache.hadoop.hbase.regionserver.HRegionServer in project hbase by apache.

the class TestScannerTimeout method test3686b.

/**
 * Make sure that no rows are lost if the scanner timeout is longer on the
 * client than the server, and the scan times out on the server but not the
 * client.
 * @throws Exception
 */
@Test
public void test3686b() throws Exception {
    LOG.info("START ************ test3686b");
    HRegionServer rs = TEST_UTIL.getRSForFirstRegionInTable(TABLE_NAME);
    Scan scan = new Scan();
    scan.setCaching(SCANNER_CACHING);
    // Set a very high timeout, we want to test what happens when a RS
    // fails but the region is recovered before the lease times out.
    // Since the RS is already created, this conf is client-side only for
    // this new table
    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
    conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, SCANNER_TIMEOUT * 100);
    Connection connection = ConnectionFactory.createConnection(conf);
    Table higherScanTimeoutTable = connection.getTable(TABLE_NAME);
    ResultScanner r = higherScanTimeoutTable.getScanner(scan);
    int count = 1;
    r.next();
    // Sleep, allowing the scan to timeout on the server but not on the client.
    Thread.sleep(SCANNER_TIMEOUT + 2000);
    while (r.next() != null) {
        count++;
    }
    assertEquals(NB_ROWS, count);
    r.close();
    higherScanTimeoutTable.close();
    connection.close();
    LOG.info("END ************ END test3686b");
}

Example 80 with HRegionServer

use of org.apache.hadoop.hbase.regionserver.HRegionServer in project hbase by apache.

the class TestHBCKSCP method test.

@Test
public void test() throws Exception {
    // we are about to do one for it?
    SingleProcessHBaseCluster cluster = this.util.getHBaseCluster();
    // Assert that we have three RegionServers. Test depends on there being multiple.
    assertEquals(RS_COUNT, cluster.getLiveRegionServerThreads().size());
    int count;
    try (Table table = createTable(TableName.valueOf(this.name.getMethodName()))) {
        // Load the table with a bit of data so some logs to split and some edits in each region.
        this.util.loadTable(table, HBaseTestingUtil.COLUMNS[0]);
        count = util.countRows(table);
    }
    assertTrue("expected some rows", count > 0);
    // Make the test easier by not working on server hosting meta...
    // Find another RS. Purge it from Master memory w/o running SCP (if
    // SCP runs, it will clear entries from hbase:meta which frustrates
    // our attempt at manufacturing 'Unknown Servers' condition).
    int metaIndex = this.util.getMiniHBaseCluster().getServerWithMeta();
    int rsIndex = (metaIndex + 1) % RS_COUNT;
    ServerName rsServerName = cluster.getRegionServer(rsIndex).getServerName();
    HMaster master = cluster.getMaster();
    // Get a Region that is on the server.
    RegionInfo rsRI = master.getAssignmentManager().getRegionsOnServer(rsServerName).get(0);
    Result r = MetaTableAccessor.getRegionResult(master.getConnection(), rsRI.getRegionName());
    // Assert region is OPEN.
    assertEquals(RegionState.State.OPEN.toString(), Bytes.toString(r.getValue(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER)));
    ServerName serverName = CatalogFamilyFormat.getServerName(r, 0);
    assertEquals(rsServerName, serverName);
    // moveFrom adds to dead servers and adds it to processing list only we will
    // not be processing this server 'normally'. Remove it from processing by
    // calling 'finish' and then remove it from dead servers so rsServerName
    // becomes an 'Unknown Server' even though it is still around.
    LOG.info("Killing {}", rsServerName);
    cluster.killRegionServer(rsServerName);
    master.getServerManager().moveFromOnlineToDeadServers(rsServerName);
    master.getServerManager().getDeadServers().removeDeadServer(rsServerName);
    master.getAssignmentManager().getRegionStates().removeServer(rsServerName);
    // Kill the server. Nothing should happen since an 'Unknown Server' as far
    // as the Master is concerned; i.e. no SCP.
    HRegionServer hrs = cluster.getRegionServer(rsServerName);
    while (!hrs.isStopped()) {
        Threads.sleep(10);
    }
    LOG.info("Dead {}", rsServerName);
    // Now assert still references in hbase:meta to the 'dead' server -- they haven't been
    // cleaned up by an SCP or by anything else.
    assertTrue(searchMeta(master, rsServerName));
    // Assert region is OPEN on dead server still.
    r = MetaTableAccessor.getRegionResult(master.getConnection(), rsRI.getRegionName());
    assertEquals(RegionState.State.OPEN.toString(), Bytes.toString(r.getValue(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER)));
    serverName = CatalogFamilyFormat.getServerName(r, 0);
    assertNotNull(cluster.getRegionServer(serverName));
    assertEquals(rsServerName, serverName);
    // I now have 'Unknown Server' references in hbase:meta; i.e. Server references
    // with no corresponding SCP. Queue one.
    long pid = scheduleHBCKSCP(rsServerName, master);
    assertNotEquals(Procedure.NO_PROC_ID, pid);
    while (master.getMasterProcedureExecutor().getActiveProcIds().contains(pid)) {
        Threads.sleep(10);
    }
    // After SCP, assert region is OPEN on new server.
    r = MetaTableAccessor.getRegionResult(master.getConnection(), rsRI.getRegionName());
    assertEquals(RegionState.State.OPEN.toString(), Bytes.toString(r.getValue(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER)));
    serverName = CatalogFamilyFormat.getServerName(r, 0);
    assertNotNull(cluster.getRegionServer(serverName));
    assertNotEquals(rsServerName, serverName);
    // Make sure no mention of old server post SCP.
    assertFalse(searchMeta(master, rsServerName));
}