Examples with RegionServerThread org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread used on opensource projects

Example 21 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestRegionReplicas method testVerifySecondaryAbilityToReadWithOnFiles.

@Test(timeout = 300000)
public void testVerifySecondaryAbilityToReadWithOnFiles() throws Exception {
    // disable the store file refresh chore (we do this by hand)
    HTU.getConfiguration().setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 0);
    restartRegionServer();
    try {
        LOG.info("Opening the secondary region " + hriSecondary.getEncodedName());
        openRegion(HTU, getRS(), hriSecondary);
        // load some data to primary
        LOG.info("Loading data to primary region");
        for (int i = 0; i < 3; ++i) {
            HTU.loadNumericRows(table, f, i * 1000, (i + 1) * 1000);
            Region region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName());
            region.flush(true);
        }
        Region primaryRegion = getRS().getFromOnlineRegions(hriPrimary.getEncodedName());
        Assert.assertEquals(3, primaryRegion.getStore(f).getStorefilesCount());
        // Refresh store files on the secondary
        Region secondaryRegion = getRS().getFromOnlineRegions(hriSecondary.getEncodedName());
        secondaryRegion.getStore(f).refreshStoreFiles();
        Assert.assertEquals(3, secondaryRegion.getStore(f).getStorefilesCount());
        // force compaction
        LOG.info("Force Major compaction on primary region " + hriPrimary);
        primaryRegion.compact(true);
        Assert.assertEquals(1, primaryRegion.getStore(f).getStorefilesCount());
        List<RegionServerThread> regionServerThreads = HTU.getMiniHBaseCluster().getRegionServerThreads();
        HRegionServer hrs = null;
        for (RegionServerThread rs : regionServerThreads) {
            if (rs.getRegionServer().getOnlineRegion(primaryRegion.getRegionInfo().getRegionName()) != null) {
                hrs = rs.getRegionServer();
                break;
            }
        }
        CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, hrs, false);
        cleaner.chore();
        // scan all the hfiles on the secondary.
        // since there are no read on the secondary when we ask locations to
        // the NN a FileNotFound exception will be returned and the FileLink
        // should be able to deal with it giving us all the result we expect.
        int keys = 0;
        int sum = 0;
        for (StoreFile sf : secondaryRegion.getStore(f).getStorefiles()) {
            // Our file does not exist anymore. was moved by the compaction above.
            LOG.debug(getRS().getFileSystem().exists(sf.getPath()));
            Assert.assertFalse(getRS().getFileSystem().exists(sf.getPath()));
            HFileScanner scanner = sf.getReader().getScanner(false, false);
            scanner.seekTo();
            do {
                keys++;
                Cell cell = scanner.getCell();
                sum += Integer.parseInt(Bytes.toString(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
            } while (scanner.next());
        }
        Assert.assertEquals(3000, keys);
        Assert.assertEquals(4498500, sum);
    } finally {
        HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000);
        closeRegion(HTU, getRS(), hriSecondary);
    }
}

Example 22 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestDistributedLogSplitting method testThreeRSAbort.

@Test(timeout = 300000)
public void testThreeRSAbort() throws Exception {
    LOG.info("testThreeRSAbort");
    final int NUM_REGIONS_TO_CREATE = 40;
    final int NUM_ROWS_PER_REGION = 100;
    // NUM_RS=6.
    startCluster(NUM_RS);
    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "distributed log splitting test", null);
    Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
    try {
        populateDataInTable(NUM_ROWS_PER_REGION, "family");
        List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
        assertEquals(NUM_RS, rsts.size());
        rsts.get(0).getRegionServer().abort("testing");
        rsts.get(1).getRegionServer().abort("testing");
        rsts.get(2).getRegionServer().abort("testing");
        long start = EnvironmentEdgeManager.currentTime();
        while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) {
            if (EnvironmentEdgeManager.currentTime() - start > 60000) {
                assertTrue(false);
            }
            Thread.sleep(200);
        }
        start = EnvironmentEdgeManager.currentTime();
        while (HBaseTestingUtility.getAllOnlineRegions(cluster).size() < (NUM_REGIONS_TO_CREATE + 1)) {
            if (EnvironmentEdgeManager.currentTime() - start > 60000) {
                assertTrue("Timedout", false);
            }
            Thread.sleep(200);
        }
        // wait for all regions are fully recovered
        TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(zkw.znodePaths.recoveringRegionsZNode, false);
                return (recoveringRegions != null && recoveringRegions.isEmpty());
            }
        });
        assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION, TEST_UTIL.countRows(ht));
    } finally {
        if (ht != null)
            ht.close();
        if (zkw != null)
            zkw.close();
    }
}

Example 23 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestDistributedLogSplitting method populateDataInTable.

void populateDataInTable(int nrows, String fname) throws Exception {
    byte[] family = Bytes.toBytes(fname);
    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
    assertEquals(NUM_RS, rsts.size());
    for (RegionServerThread rst : rsts) {
        HRegionServer hrs = rst.getRegionServer();
        List<HRegionInfo> hris = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
        for (HRegionInfo hri : hris) {
            if (hri.getTable().isSystemTable()) {
                continue;
            }
            LOG.debug("adding data to rs = " + rst.getName() + " region = " + hri.getRegionNameAsString());
            Region region = hrs.getOnlineRegion(hri.getRegionName());
            assertTrue(region != null);
            putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family);
        }
    }
    for (MasterThread mt : cluster.getLiveMasterThreads()) {
        HRegionServer hrs = mt.getMaster();
        List<HRegionInfo> hris;
        try {
            hris = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
        } catch (ServerNotRunningYetException e) {
            // It's ok: this master may be a backup. Ignored.
            continue;
        }
        for (HRegionInfo hri : hris) {
            if (hri.getTable().isSystemTable()) {
                continue;
            }
            LOG.debug("adding data to rs = " + mt.getName() + " region = " + hri.getRegionNameAsString());
            Region region = hrs.getOnlineRegion(hri.getRegionName());
            assertTrue(region != null);
            putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family);
        }
    }
}

Example 24 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestDistributedLogSplitting method testLogReplayTwoSequentialRSDown.

@Ignore("DLR is broken by HBASE-12751")
@Test(timeout = 300000)
public void testLogReplayTwoSequentialRSDown() throws Exception {
    LOG.info("testRecoveredEditsReplayTwoSequentialRSDown");
    conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
    startCluster(NUM_RS);
    final int NUM_REGIONS_TO_CREATE = 40;
    final int NUM_LOG_LINES = 1000;
    // turn off load balancing to prevent regions from moving around otherwise
    // they will consume recovered.edits
    master.balanceSwitch(false);
    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
    final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
    Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
    try {
        List<HRegionInfo> regions = null;
        HRegionServer hrs1 = findRSToKill(false, "table");
        regions = ProtobufUtil.getOnlineRegions(hrs1.getRSRpcServices());
        makeWAL(hrs1, regions, "table", "family", NUM_LOG_LINES, 100);
        // abort RS1
        LOG.info("Aborting region server: " + hrs1.getServerName());
        hrs1.abort("testing");
        // wait for abort completes
        TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
            }
        });
        // wait for regions come online
        TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() >= (NUM_REGIONS_TO_CREATE + 1));
            }
        });
        // sleep a little bit in order to interrupt recovering in the middle
        Thread.sleep(300);
        // abort second region server
        rsts = cluster.getLiveRegionServerThreads();
        HRegionServer hrs2 = rsts.get(0).getRegionServer();
        LOG.info("Aborting one more region server: " + hrs2.getServerName());
        hrs2.abort("testing");
        // wait for abort completes
        TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 2));
            }
        });
        // wait for regions come online
        TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() >= (NUM_REGIONS_TO_CREATE + 1));
            }
        });
        // wait for all regions are fully recovered
        TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(zkw.znodePaths.recoveringRegionsZNode, false);
                return (recoveringRegions != null && recoveringRegions.isEmpty());
            }
        });
        assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
    } finally {
        if (ht != null)
            ht.close();
        if (zkw != null)
            zkw.close();
    }
}

Example 25 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestDistributedLogSplitting method testMarkRegionsRecoveringInZK.

@Ignore("DLR is broken by HBASE-12751")
@Test(timeout = 300000)
public void testMarkRegionsRecoveringInZK() throws Exception {
    LOG.info("testMarkRegionsRecoveringInZK");
    conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true);
    startCluster(NUM_RS);
    master.balanceSwitch(false);
    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
    final ZooKeeperWatcher zkw = master.getZooKeeper();
    Table ht = installTable(zkw, "table", "family", 40);
    try {
        final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager();
        Set<HRegionInfo> regionSet = new HashSet<>();
        HRegionInfo region = null;
        HRegionServer hrs = null;
        ServerName firstFailedServer = null;
        ServerName secondFailedServer = null;
        for (int i = 0; i < NUM_RS; i++) {
            hrs = rsts.get(i).getRegionServer();
            List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
            if (regions.isEmpty())
                continue;
            region = regions.get(0);
            regionSet.add(region);
            firstFailedServer = hrs.getServerName();
            secondFailedServer = rsts.get((i + 1) % NUM_RS).getRegionServer().getServerName();
            break;
        }
        slm.markRegionsRecovering(firstFailedServer, regionSet);
        slm.markRegionsRecovering(secondFailedServer, regionSet);
        List<String> recoveringRegions = ZKUtil.listChildrenNoWatch(zkw, ZKUtil.joinZNode(zkw.znodePaths.recoveringRegionsZNode, region.getEncodedName()));
        assertEquals(recoveringRegions.size(), 2);
        // wait for splitLogWorker to mark them up because there is no WAL files recorded in ZK
        final HRegionServer tmphrs = hrs;
        TEST_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                return (tmphrs.getRecoveringRegions().isEmpty());
            }
        });
    } finally {
        if (ht != null)
            ht.close();
        if (zkw != null)
            zkw.close();
    }
}