Examples with RegionServerThread org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread used on opensource projects

Example 16 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestVisibilityLabels method testVisibilityLabelsOnKillingOfRSContainingLabelsTable.

@Test
public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
    int liveRS = 0;
    for (RegionServerThread rsThreads : regionServerThreads) {
        if (!rsThreads.getRegionServer().isAborted()) {
            liveRS++;
        }
    }
    if (liveRS == 1) {
        TEST_UTIL.getHBaseCluster().startRegionServer();
    }
    Thread t1 = new Thread() {

        public void run() {
            List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
            for (RegionServerThread rsThread : regionServerThreads) {
                List<Region> onlineRegions = rsThread.getRegionServer().getOnlineRegions(LABELS_TABLE_NAME);
                if (onlineRegions.size() > 0) {
                    rsThread.getRegionServer().abort("Aborting ");
                    killedRS = true;
                    break;
                }
            }
        }
    };
    t1.start();
    final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
    Thread t = new Thread() {

        public void run() {
            try {
                while (!killedRS) {
                    Thread.sleep(1);
                }
                createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE);
            } catch (Exception e) {
            }
        }
    };
    t.start();
    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
    while (!killedRS) {
        Thread.sleep(10);
    }
    regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
    for (RegionServerThread rsThread : regionServerThreads) {
        while (true) {
            if (!rsThread.getRegionServer().isAborted()) {
                List<Region> onlineRegions = rsThread.getRegionServer().getOnlineRegions(LABELS_TABLE_NAME);
                if (onlineRegions.size() > 0) {
                    break;
                } else {
                    Thread.sleep(10);
                }
            } else {
                break;
            }
        }
    }
    TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
    t.join();
    try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
        Scan s = new Scan();
        s.setAuthorizations(new Authorizations(SECRET));
        ResultScanner scanner = table.getScanner(s);
        Result[] next = scanner.next(3);
        assertTrue(next.length == 1);
    }
}

Example 17 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestCorruptedRegionStoreFile method evictHFileCache.

private void evictHFileCache(final Path hfile) throws Exception {
    for (RegionServerThread rst : UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
        HRegionServer rs = rst.getRegionServer();
        rs.getCacheConfig().getBlockCache().evictBlocksByHfileName(hfile.getName());
    }
}

Example 18 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestRegionReplicaFailover method testSecondaryRegionKillWhilePrimaryIsAcceptingWrites.

/**
   * Tests the case where there are 3 region replicas and the primary is continuously accepting
   * new writes while one of the secondaries is killed. Verification is done for both of the
   * secondary replicas.
   */
@Test(timeout = 120000)
public void testSecondaryRegionKillWhilePrimaryIsAcceptingWrites() throws Exception {
    try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
        Table table = connection.getTable(htd.getTableName());
        Admin admin = connection.getAdmin()) {
        // start a thread to do the loading of primary
        // start with some base
        HTU.loadNumericRows(table, fam, 0, 1000);
        admin.flush(table.getName());
        HTU.loadNumericRows(table, fam, 1000, 2000);
        final AtomicReference<Throwable> ex = new AtomicReference<>(null);
        final AtomicBoolean done = new AtomicBoolean(false);
        final AtomicInteger key = new AtomicInteger(2000);
        Thread loader = new Thread() {

            @Override
            public void run() {
                while (!done.get()) {
                    try {
                        HTU.loadNumericRows(table, fam, key.get(), key.get() + 1000);
                        key.addAndGet(1000);
                    } catch (Throwable e) {
                        ex.compareAndSet(null, e);
                    }
                }
            }
        };
        loader.start();
        Thread aborter = new Thread() {

            @Override
            public void run() {
                try {
                    boolean aborted = false;
                    for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
                        for (Region r : rs.getRegionServer().getOnlineRegions(htd.getTableName())) {
                            if (r.getRegionInfo().getReplicaId() == 1) {
                                LOG.info("Aborting region server hosting secondary region replica");
                                rs.getRegionServer().abort("for test");
                                aborted = true;
                            }
                        }
                    }
                    assertTrue(aborted);
                } catch (Throwable e) {
                    ex.compareAndSet(null, e);
                }
            }

            ;
        };
        aborter.start();
        aborter.join();
        done.set(true);
        loader.join();
        assertNull(ex.get());
        // assert that the test is working as designed
        assertTrue(key.get() > 1000);
        LOG.info("Loaded up to key :" + key.get());
        verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 0, 30000);
        verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 1, 30000);
        verifyNumericRowsWithTimeout(table, fam, 0, key.get(), 2, 30000);
    }
    // restart the region server
    HTU.getMiniHBaseCluster().startRegionServer();
}

Example 19 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class MiniHBaseCluster method startRegionServer.

/**
   * Starts a region server thread running
   *
   * @throws IOException
   * @return New RegionServerThread
   */
public JVMClusterUtil.RegionServerThread startRegionServer() throws IOException {
    final Configuration newConf = HBaseConfiguration.create(conf);
    User rsUser = HBaseTestingUtility.getDifferentUser(newConf, ".hfs." + index++);
    JVMClusterUtil.RegionServerThread t = null;
    try {
        t = hbaseCluster.addRegionServer(newConf, hbaseCluster.getRegionServers().size(), rsUser);
        t.start();
        t.waitForServerOnline();
    } catch (InterruptedException ie) {
        throw new IOException("Interrupted adding regionserver to cluster", ie);
    }
    return t;
}

Example 20 with RegionServerThread

use of org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread in project hbase by apache.

the class TestIOFencing method doTest.

public void doTest(Class<?> regionClass, MemoryCompactionPolicy policy) throws Exception {
    Configuration c = TEST_UTIL.getConfiguration();
    // Insert our custom region
    c.setClass(HConstants.REGION_IMPL, regionClass, HRegion.class);
    // Encourage plenty of flushes
    c.setLong("hbase.hregion.memstore.flush.size", 25000);
    c.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, ConstantSizeRegionSplitPolicy.class.getName());
    // Only run compaction when we tell it to
    c.setInt("hbase.hstore.compaction.min", 1);
    c.setInt("hbase.hstore.compactionThreshold", 1000);
    c.setLong("hbase.hstore.blockingStoreFiles", 1000);
    // Compact quickly after we tell it to!
    c.setInt("hbase.regionserver.thread.splitcompactcheckfrequency", 1000);
    c.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy));
    LOG.info("Starting mini cluster");
    TEST_UTIL.startMiniCluster(1);
    CompactionBlockerRegion compactingRegion = null;
    Admin admin = null;
    try {
        LOG.info("Creating admin");
        admin = TEST_UTIL.getConnection().getAdmin();
        LOG.info("Creating table");
        TEST_UTIL.createTable(TABLE_NAME, FAMILY);
        Table table = TEST_UTIL.getConnection().getTable(TABLE_NAME);
        LOG.info("Loading test table");
        // Find the region
        List<HRegion> testRegions = TEST_UTIL.getMiniHBaseCluster().findRegionsForTable(TABLE_NAME);
        assertEquals(1, testRegions.size());
        compactingRegion = (CompactionBlockerRegion) testRegions.get(0);
        LOG.info("Blocking compactions");
        compactingRegion.stopCompactions();
        long lastFlushTime = compactingRegion.getEarliestFlushTimeForAllStores();
        // Load some rows
        TEST_UTIL.loadNumericRows(table, FAMILY, 0, FIRST_BATCH_COUNT);
        // add a compaction from an older (non-existing) region to see whether we successfully skip
        // those entries
        HRegionInfo oldHri = new HRegionInfo(table.getName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
        CompactionDescriptor compactionDescriptor = ProtobufUtil.toCompactionDescriptor(oldHri, FAMILY, Lists.newArrayList(new Path("/a")), Lists.newArrayList(new Path("/b")), new Path("store_dir"));
        WALUtil.writeCompactionMarker(compactingRegion.getWAL(), ((HRegion) compactingRegion).getReplicationScope(), oldHri, compactionDescriptor, compactingRegion.getMVCC());
        // Wait till flush has happened, otherwise there won't be multiple store files
        long startWaitTime = System.currentTimeMillis();
        while (compactingRegion.getEarliestFlushTimeForAllStores() <= lastFlushTime || compactingRegion.countStoreFiles() <= 1) {
            LOG.info("Waiting for the region to flush " + compactingRegion.getRegionInfo().getRegionNameAsString());
            Thread.sleep(1000);
            assertTrue("Timed out waiting for the region to flush", System.currentTimeMillis() - startWaitTime < 30000);
        }
        assertTrue(compactingRegion.countStoreFiles() > 1);
        final byte[] REGION_NAME = compactingRegion.getRegionInfo().getRegionName();
        LOG.info("Asking for compaction");
        admin.majorCompact(TABLE_NAME);
        LOG.info("Waiting for compaction to be about to start");
        compactingRegion.waitForCompactionToBlock();
        LOG.info("Starting a new server");
        RegionServerThread newServerThread = TEST_UTIL.getMiniHBaseCluster().startRegionServer();
        final HRegionServer newServer = newServerThread.getRegionServer();
        LOG.info("Killing region server ZK lease");
        TEST_UTIL.expireRegionServerSession(0);
        CompactionBlockerRegion newRegion = null;
        startWaitTime = System.currentTimeMillis();
        LOG.info("Waiting for the new server to pick up the region " + Bytes.toString(REGION_NAME));
        // wait for region to be assigned and to go out of log replay if applicable
        Waiter.waitFor(c, 60000, new Waiter.Predicate<Exception>() {

            @Override
            public boolean evaluate() throws Exception {
                Region newRegion = newServer.getOnlineRegion(REGION_NAME);
                return newRegion != null && !newRegion.isRecovering();
            }
        });
        newRegion = (CompactionBlockerRegion) newServer.getOnlineRegion(REGION_NAME);
        // After compaction of old region finishes on the server that was going down, make sure that
        // all the files we expect are still working when region is up in new location.
        FileSystem fs = newRegion.getFilesystem();
        for (String f : newRegion.getStoreFileList(new byte[][] { FAMILY })) {
            assertTrue("After compaction, does not exist: " + f, fs.exists(new Path(f)));
        }
        LOG.info("Allowing compaction to proceed");
        compactingRegion.allowCompactions();
        while (compactingRegion.compactCount == 0) {
            Thread.sleep(1000);
        }
        // The server we killed stays up until the compaction that was started before it was killed completes.  In logs
        // you should see the old regionserver now going down.
        LOG.info("Compaction finished");
        // If we survive the split keep going...
        // Now we make sure that the region isn't totally confused.  Load up more rows.
        TEST_UTIL.loadNumericRows(table, FAMILY, FIRST_BATCH_COUNT, FIRST_BATCH_COUNT + SECOND_BATCH_COUNT);
        admin.majorCompact(TABLE_NAME);
        startWaitTime = System.currentTimeMillis();
        while (newRegion.compactCount == 0) {
            Thread.sleep(1000);
            assertTrue("New region never compacted", System.currentTimeMillis() - startWaitTime < 180000);
        }
        if (policy == MemoryCompactionPolicy.EAGER) {
            assertTrue(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT >= TEST_UTIL.countRows(table));
        } else {
            assertEquals(FIRST_BATCH_COUNT + SECOND_BATCH_COUNT, TEST_UTIL.countRows(table));
        }
    } finally {
        if (compactingRegion != null) {
            compactingRegion.allowCompactions();
        }
        admin.close();
        TEST_UTIL.shutdownMiniCluster();
    }
}