Examples with RegionServerThread org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread used on opensource projects

Example 31 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class MiniHBaseCluster method startRegionServer.

/**
   * Starts a region server thread running
   *
   * @throws IOException
   * @return New RegionServerThread
   */
public JVMClusterUtil.RegionServerThread startRegionServer() throws IOException {
    final Configuration newConf = HBaseConfiguration.create(conf);
    User rsUser = HBaseTestingUtility.getDifferentUser(newConf, ".hfs." + index++);
    JVMClusterUtil.RegionServerThread t = null;
    try {
        t = hbaseCluster.addRegionServer(newConf, hbaseCluster.getRegionServers().size(), rsUser);
        t.start();
        t.waitForServerOnline();
    } catch (InterruptedException ie) {
        throw new IOException("Interrupted adding regionserver to cluster", ie);
    }
    return t;
}

Example 32 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestIOFencing method doTest.

public void doTest(Class<?> regionClass, MemoryCompactionPolicy policy) throws Exception {
    Configuration c = TEST_UTIL.getConfiguration();
    // Insert our custom region
    c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class);
    // Encourage plenty of flushes
    c.setLong("hbase.hregion.memstore.flush.size", 25000);
    c.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
    // Only run compaction when we tell it to
    c.setInt("hbase.hstore.compaction.min", 1);
    c.setInt("hbase.hstore.compactionThreshold", 1000);
    c.setLong("hbase.hstore.blockingStoreFiles", 1000);
    // Compact quickly after we tell it to!
    c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000);
    c.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy));
    LOG.info("Starting mini cluster");
    TEST_UTIL.startMiniCluster(1);
    CompactionBlockerRegion compactingRegion = null;
    Admin admin = null;
    try {
        LOG.info("Creating admin");
        admin = TEST_UTIL.getConnection().getAdmin();
        LOG.info("Creating table");
        TEST_UTIL.createTable(TABLE_NAME, FAMILY);
        Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME);
        LOG.info("Loading test table");
        // Find the region
        List<HRegion> testRegions = TEST_UTIL.getMiniHBaseCluster().findRegionsForTable(TABLE_NAME);
        assertEquals(1, testRegions.size());
        compactingRegion = (CompactionBlockerRegion) testRegions.get(0);
        LOG.info("Blocking compactions");
        compactingRegion.stopCompactions();
        long lastFlushTime = compactingRegion.getEarliestFlushTimeForAllStores();
        // Load some rows
        TEST_UTIL.loadNumericRows(table, FAMILY, 0, FIRST_BATCH_COUNT);
        // add a compaction from an older (non-existing) region to see whether we successfully skip
        // those entries
        HRegionInfo oldHri = new HRegionInfo(table.getName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
        CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(oldHri, FAMILY, Lists.newArrayList(new Path("/a")), Lists.newArrayList(new Path("/b")), new Path("store_dir"));
        WALUtil.writeCompactionMarker(compactingRegion.getWAL(), ((HRegion) compactingRegion).getReplicationScope(), oldHri, compactionDescriptor, compactingRegion.getMVCC());
        // Wait till flush has happened, otherwise there won't be multiple store files
        long startWaitTime = System.currentTimeMillis();
        while (compactingRegion.getEarliestFlushTimeForAllStores() <= lastFlushTime || compactingRegion.countStoreFiles() <= 1) {
            LOG.info("Waiting for the region to flush " + compactingRegion.getRegionInfo().getRegionNameAsString());
            Thread.sleep(1000);
            assertTrue("Timed out waiting for the region to flush", System.currentTimeMillis() - startWaitTime < 30000);
        }
        assertTrue(compactingRegion.countStoreFiles() > 1);
        final byte[] REGION_NAME = compactingRegion.getRegionInfo().getRegionName();
        LOG.info("Asking for compaction");
        admin.majorCompact(TABLE_NAME);
        LOG.info("Waiting for compaction to be about to start");
        compactingRegion.waitForCompactionToBlock();
        LOG.info("Starting a new server");
        RegionServerThread newServerThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer();
        final HRegionServer newServer = newServerThread.getRegionServer();
        LOG.info("Killing region server ZK lease");
        TEST_UTIL.expireRegionServerSession(0);
        CompactionBlockerRegion newRegion = null;
        startWaitTime = System.currentTimeMillis();
        LOG.info("Waiting for the new server to pick up the region " + Bytes.toString(REGION_NAME));
        // wait for region to be assigned and to go out of log replay if applicable
        Waiter.waitFor(c, 60000, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                Region newRegion = newServer.getOnlineRegion(REGION_NAME);
                return newRegion != null && !newRegion.isRecovering();
            }
        });
        newRegion = (CompactionBlockerRegion) newServer.getOnlineRegion(REGION_NAME);
        // After compaction of old region finishes on the server that was going down, make sure that
        // all the files we expect are still working when region is up in new location.
        FileSystem fs = newRegion.getFilesystem();
        for (String f : newRegion.getStoreFileList(new byte[][] { FAMILY })) {
            assertTrue("After compaction, does not exist: " + f, fs.exists(new Path(f)));
        }
        LOG.info("Allowing compaction to proceed");
        compactingRegion.allowCompactions();
        while (compactingRegion.compactCount == 0) {
            Thread.sleep(1000);
        }
        // The server we killed stays up until the compaction that was started before it was killed completes.  In logs
        // you should see the old regionserver now going down.
        LOG.info("Compaction finished");
        // If we survive the split keep going...
        // Now we make sure that the region isn't totally confused.  Load up more rows.
        TEST_UTIL.loadNumericRows(table, FAMILY, FIRST_BATCH_COUNT, FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
        admin.majorCompact(TABLE_NAME);
        startWaitTime = System.currentTimeMillis();
        while (newRegion.compactCount == 0) {
            Thread.sleep(1000);
            assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000);
        }
        if (policy == MemoryCompactionPolicy.EAGER) {
            assertTrue(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT >= TEST_UTIL.countRows(table));
        } else {
            assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table));
        }
    } finally {
        if (compactingRegion != null) {
            compactingRegion.allowCompactions();
        }
        admin.close();
        TEST_UTIL.shutdownMiniCluster();
    }
}

Example 33 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestRegionMergeTransactionOnCluster method testCleanMergeReference.

@Test
public void testCleanMergeReference() throws Exception {
    LOG.info("Starting " + name.getMethodName());
    ADMIN.enableCatalogJanitor(false);
    try {
        final TableName tableName = TableName.valueOf(name.getMethodName());
        // Create table and load data.
        Table table = createTableAndLoadData(MASTER, tableName);
        // Merge 1st and 2nd region
        mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);
        verifyRowCount(table, ROWSIZE);
        table.close();
        List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor.getTableRegionsAndLocations(MASTER.getConnection(), tableName);
        HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
        HTableDescriptor tableDescriptor = MASTER.getTableDescriptors().get(tableName);
        Result mergedRegionResult = MetaTableAccessor.getRegionResult(MASTER.getConnection(), mergedRegionInfo.getRegionName());
        // contains merge reference in META
        assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null);
        assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER) != null);
        // merging regions' directory are in the file system all the same
        PairOfSameType<HRegionInfo> p = MetaTableAccessor.getMergeRegions(mergedRegionResult);
        HRegionInfo regionA = p.getFirst();
        HRegionInfo regionB = p.getSecond();
        FileSystem fs = MASTER.getMasterFileSystem().getFileSystem();
        Path rootDir = MASTER.getMasterFileSystem().getRootDir();
        Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
        Path regionAdir = new Path(tabledir, regionA.getEncodedName());
        Path regionBdir = new Path(tabledir, regionB.getEncodedName());
        assertTrue(fs.exists(regionAdir));
        assertTrue(fs.exists(regionBdir));
        HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
        HRegionFileSystem hrfs = new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
        int count = 0;
        for (HColumnDescriptor colFamily : columnFamilies) {
            count += hrfs.getStoreFiles(colFamily.getName()).size();
        }
        ADMIN.compactRegion(mergedRegionInfo.getRegionName());
        // clean up the merged region store files
        // wait until merged region have reference file
        long timeout = System.currentTimeMillis() + waitTime;
        int newcount = 0;
        while (System.currentTimeMillis() < timeout) {
            for (HColumnDescriptor colFamily : columnFamilies) {
                newcount += hrfs.getStoreFiles(colFamily.getName()).size();
            }
            if (newcount > count) {
                break;
            }
            Thread.sleep(50);
        }
        assertTrue(newcount > count);
        List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
        for (RegionServerThread rs : regionServerThreads) {
            CompactedHFilesDischarger cleaner = new CompactedHFilesDischarger(100, null, rs.getRegionServer(), false);
            cleaner.chore();
            Thread.sleep(1000);
        }
        while (System.currentTimeMillis() < timeout) {
            int newcount1 = 0;
            for (HColumnDescriptor colFamily : columnFamilies) {
                newcount1 += hrfs.getStoreFiles(colFamily.getName()).size();
            }
            if (newcount1 <= 1) {
                break;
            }
            Thread.sleep(50);
        }
        // run CatalogJanitor to clean merge references in hbase:meta and archive the
        // files of merging regions
        int cleaned = 0;
        while (cleaned == 0) {
            cleaned = ADMIN.runCatalogScan();
            LOG.debug("catalog janitor returned " + cleaned);
            Thread.sleep(50);
        }
        assertFalse(regionAdir.toString(), fs.exists(regionAdir));
        assertFalse(regionBdir.toString(), fs.exists(regionBdir));
        assertTrue(cleaned > 0);
        mergedRegionResult = MetaTableAccessor.getRegionResult(TEST_UTIL.getConnection(), mergedRegionInfo.getRegionName());
        assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null);
        assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEB_QUALIFIER) != null);
    } finally {
        ADMIN.enableCatalogJanitor(true);
    }
}

Example 34 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project phoenix by apache.

the class TestWALRecoveryCaching method testWaitsOnIndexRegionToReload.

//TODO: Jesse to fix
@SuppressWarnings("deprecation")
@Ignore("Configuration issue - valid test, just needs fixing")
@Test
public void testWaitsOnIndexRegionToReload() throws Exception {
    HBaseTestingUtility util = new HBaseTestingUtility();
    Configuration conf = util.getConfiguration();
    setUpConfigForMiniCluster(conf);
    // setup other useful stats
    IndexTestingUtils.setupConfig(conf);
    conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false);
    // make sure everything is setup correctly
    IndexManagementUtil.ensureMutableIndexingCorrectlyConfigured(conf);
    // start the cluster with 2 rs
    util.startMiniCluster(2);
    HBaseAdmin admin = util.getHBaseAdmin();
    // setup the index
    byte[] family = Bytes.toBytes("family");
    byte[] qual = Bytes.toBytes("qualifier");
    byte[] nonIndexedFamily = Bytes.toBytes("nonIndexedFamily");
    String indexedTableName = getIndexTableName();
    ColumnGroup columns = new ColumnGroup(indexedTableName);
    columns.add(new CoveredColumn(family, qual));
    CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
    builder.addIndexGroup(columns);
    // create the primary table w/ indexing enabled
    HTableDescriptor primaryTable = new HTableDescriptor(testTable.getTableName());
    primaryTable.addFamily(new HColumnDescriptor(family));
    primaryTable.addFamily(new HColumnDescriptor(nonIndexedFamily));
    builder.addArbitraryConfigForTesting(Indexer.RecoveryFailurePolicyKeyForTesting, ReleaseLatchOnFailurePolicy.class.getName());
    builder.build(primaryTable);
    admin.createTable(primaryTable);
    // create the index table
    HTableDescriptor indexTableDesc = new HTableDescriptor(Bytes.toBytes(getIndexTableName()));
    indexTableDesc.addCoprocessor(IndexTableBlockingReplayObserver.class.getName());
    CoveredColumnIndexer.createIndexTable(admin, indexTableDesc);
    // figure out where our tables live
    ServerName shared = ensureTablesLiveOnSameServer(util.getMiniHBaseCluster(), Bytes.toBytes(indexedTableName), testTable.getTableName());
    // load some data into the table
    Put p = new Put(Bytes.toBytes("row"));
    p.add(family, qual, Bytes.toBytes("value"));
    HTable primary = new HTable(conf, testTable.getTableName());
    primary.put(p);
    primary.flushCommits();
    // turn on the recovery latch
    allowIndexTableToRecover = new CountDownLatch(1);
    // kill the server where the tables live - this should trigger distributed log splitting
    // find the regionserver that matches the passed server
    List<Region> online = new ArrayList<Region>();
    online.addAll(getRegionsFromServerForTable(util.getMiniHBaseCluster(), shared, testTable.getTableName()));
    online.addAll(getRegionsFromServerForTable(util.getMiniHBaseCluster(), shared, Bytes.toBytes(indexedTableName)));
    // log all the current state of the server
    LOG.info("Current Server/Region paring: ");
    for (RegionServerThread t : util.getMiniHBaseCluster().getRegionServerThreads()) {
        // check all the conditions for the server to be done
        HRegionServer server = t.getRegionServer();
        if (server.isStopping() || server.isStopped() || server.isAborted()) {
            LOG.info("\t== Offline: " + server.getServerName());
            continue;
        }
        List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server.getRSRpcServices());
        LOG.info("\t" + server.getServerName() + " regions: " + regions);
    }
    LOG.debug("Killing server " + shared);
    util.getMiniHBaseCluster().killRegionServer(shared);
    LOG.debug("Waiting on server " + shared + "to die");
    util.getMiniHBaseCluster().waitForRegionServerToStop(shared, TIMEOUT);
    // force reassign the regions from the table
    // LOG.debug("Forcing region reassignment from the killed server: " + shared);
    // for (HRegion region : online) {
    // util.getMiniHBaseCluster().getMaster().assign(region.getRegionName());
    // }
    System.out.println(" ====== Killed shared server ==== ");
    // make a second put that (1), isn't indexed, so we can be sure of the index state and (2)
    // ensures that our table is back up
    Put p2 = new Put(p.getRow());
    p2.add(nonIndexedFamily, Bytes.toBytes("Not indexed"), Bytes.toBytes("non-indexed value"));
    primary.put(p2);
    primary.flushCommits();
    // make sure that we actually failed the write once (within a 5 minute window)
    assertTrue("Didn't find an error writing to index table within timeout!", allowIndexTableToRecover.await(ONE_MIN * 5, TimeUnit.MILLISECONDS));
    // scan the index to make sure it has the one entry, (that had to be replayed from the WAL,
    // since we hard killed the server)
    Scan s = new Scan();
    HTable index = new HTable(conf, getIndexTableName());
    ResultScanner scanner = index.getScanner(s);
    int count = 0;
    for (Result r : scanner) {
        LOG.info("Got index table result:" + r);
        count++;
    }
    assertEquals("Got an unexpected found of index rows", 1, count);
    // cleanup
    scanner.close();
    index.close();
    primary.close();
    util.shutdownMiniCluster();
}