Examples with HMaster org.apache.hadoop.hbase.master.HMaster used on opensource projects

Example 61 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class HBaseTestingUtility method predicateNoRegionsInTransition.

/**
 * Returns a {@link Predicate} for checking that there are no regions in transition in master
 */
public ExplainingPredicate<IOException> predicateNoRegionsInTransition() {
    return new ExplainingPredicate<IOException>() {

        @Override
        public String explainFailure() throws IOException {
            final RegionStates regionStates = getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
            return "found in transition: " + regionStates.getRegionsInTransition().toString();
        }

        @Override
        public boolean evaluate() throws IOException {
            HMaster master = getMiniHBaseCluster().getMaster();
            if (master == null)
                return false;
            AssignmentManager am = master.getAssignmentManager();
            if (am == null)
                return false;
            return !am.hasRegionsInTransition();
        }
    };
}

Example 62 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestHCM method testRegionCaching.

/**
   * Test that when we delete a location using the first row of a region
   * that we really delete it.
   * @throws Exception
   */
@Test
public void testRegionCaching() throws Exception {
    TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAM_NAM).close();
    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
    // test with no retry, or client cache will get updated after the first failure
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);
    Connection connection = ConnectionFactory.createConnection(conf);
    final Table table = connection.getTable(TABLE_NAME);
    TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
    Put put = new Put(ROW);
    put.addColumn(FAM_NAM, ROW, ROW);
    table.put(put);
    ConnectionImplementation conn = (ConnectionImplementation) connection;
    assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
    final int nextPort = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() + 1;
    HRegionLocation loc = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
    conn.updateCachedLocation(loc.getRegionInfo(), loc.getServerName(), ServerName.valueOf("127.0.0.1", nextPort, HConstants.LATEST_TIMESTAMP), HConstants.LATEST_TIMESTAMP);
    Assert.assertEquals(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort(), nextPort);
    conn.clearRegionCache(TABLE_NAME, ROW.clone());
    RegionLocations rl = conn.getCachedLocation(TABLE_NAME, ROW);
    assertNull("What is this location?? " + rl, rl);
    // We're now going to move the region and check that it works for the client
    // First a new put to add the location in the cache
    conn.clearRegionCache(TABLE_NAME);
    Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME));
    Put put2 = new Put(ROW);
    put2.addColumn(FAM_NAM, ROW, ROW);
    table.put(put2);
    assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
    assertNotNull(conn.getCachedLocation(TableName.valueOf(TABLE_NAME.getName()), ROW.clone()));
    TEST_UTIL.getAdmin().setBalancerRunning(false, false);
    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
    // We can wait for all regions to be online, that makes log reading easier when debugging
    TEST_UTIL.waitUntilNoRegionsInTransition();
    // Now moving the region to the second server
    HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
    byte[] regionName = toMove.getRegionInfo().getRegionName();
    byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
    // Choose the other server.
    int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
    int destServerId = (curServerId == 0 ? 1 : 0);
    HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
    HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
    ServerName destServerName = destServer.getServerName();
    // Check that we are in the expected state
    Assert.assertTrue(curServer != destServer);
    Assert.assertFalse(curServer.getServerName().equals(destServer.getServerName()));
    Assert.assertFalse(toMove.getPort() == destServerName.getPort());
    Assert.assertNotNull(curServer.getOnlineRegion(regionName));
    Assert.assertNull(destServer.getOnlineRegion(regionName));
    Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates().isRegionsInTransition());
    // Moving. It's possible that we don't have all the regions online at this point, so
    //  the test must depends only on the region we're looking at.
    LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
    TEST_UTIL.getAdmin().move(toMove.getRegionInfo().getEncodedNameAsBytes(), destServerName.getServerName().getBytes());
    while (destServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
        // wait for the move to be finished
        Thread.sleep(1);
    }
    LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
    // Check our new state.
    Assert.assertNull(curServer.getOnlineRegion(regionName));
    Assert.assertNotNull(destServer.getOnlineRegion(regionName));
    Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    // Cache was NOT updated and points to the wrong server
    Assert.assertFalse(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() == destServerName.getPort());
    // This part relies on a number of tries equals to 1.
    // We do a put and expect the cache to be updated, even if we don't retry
    LOG.info("Put starting");
    Put put3 = new Put(ROW);
    put3.addColumn(FAM_NAM, ROW, ROW);
    try {
        table.put(put3);
        Assert.fail("Unreachable point");
    } catch (RetriesExhaustedWithDetailsException e) {
        LOG.info("Put done, exception caught: " + e.getClass());
        Assert.assertEquals(1, e.getNumExceptions());
        Assert.assertEquals(1, e.getCauses().size());
        Assert.assertArrayEquals(e.getRow(0).getRow(), ROW);
        // Check that we unserialized the exception as expected
        Throwable cause = ClientExceptionsUtil.findException(e.getCause(0));
        Assert.assertNotNull(cause);
        Assert.assertTrue(cause instanceof RegionMovedException);
    }
    Assert.assertNotNull("Cached connection is null", conn.getCachedLocation(TABLE_NAME, ROW));
    Assert.assertEquals("Previous server was " + curServer.getServerName().getHostAndPort(), destServerName.getPort(), conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
    Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    // We move it back to do another test with a scan
    LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
    TEST_UTIL.getAdmin().move(toMove.getRegionInfo().getEncodedNameAsBytes(), curServer.getServerName().getServerName().getBytes());
    while (curServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
        // wait for the move to be finished
        Thread.sleep(1);
    }
    // Check our new state.
    Assert.assertNotNull(curServer.getOnlineRegion(regionName));
    Assert.assertNull(destServer.getOnlineRegion(regionName));
    LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
    // Cache was NOT updated and points to the wrong server
    Assert.assertFalse(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() == curServer.getServerName().getPort());
    Scan sc = new Scan();
    sc.setStopRow(ROW);
    sc.setStartRow(ROW);
    // The scanner takes the max retries from the connection configuration, not the table as
    // the put.
    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
    try {
        ResultScanner rs = table.getScanner(sc);
        while (rs.next() != null) {
        }
        Assert.fail("Unreachable point");
    } catch (RetriesExhaustedException e) {
        LOG.info("Scan done, expected exception caught: " + e.getClass());
    }
    // Cache is updated with the right value.
    Assert.assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
    Assert.assertEquals("Previous server was " + destServer.getServerName().getHostAndPort(), curServer.getServerName().getPort(), conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, RPC_RETRY);
    table.close();
    connection.close();
}

Example 63 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestHCM method testMulti.

@Test
public void testMulti() throws Exception {
    Table table = TEST_UTIL.createMultiRegionTable(TABLE_NAME3, FAM_NAM);
    try {
        ConnectionImplementation conn = (ConnectionImplementation) TEST_UTIL.getConnection();
        // We're now going to move the region and check that it works for the client
        // First a new put to add the location in the cache
        conn.clearRegionCache(TABLE_NAME3);
        Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME3));
        TEST_UTIL.getAdmin().setBalancerRunning(false, false);
        HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
        // We can wait for all regions to be online, that makes log reading easier when debugging
        TEST_UTIL.waitUntilNoRegionsInTransition();
        Put put = new Put(ROW_X);
        put.addColumn(FAM_NAM, ROW_X, ROW_X);
        table.put(put);
        // Now moving the region to the second server
        HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME3, ROW_X).getRegionLocation();
        byte[] regionName = toMove.getRegionInfo().getRegionName();
        byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
        // Choose the other server.
        int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
        int destServerId = (curServerId == 0 ? 1 : 0);
        HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
        HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
        ServerName destServerName = destServer.getServerName();
        //find another row in the cur server that is less than ROW_X
        List<Region> regions = curServer.getOnlineRegions(TABLE_NAME3);
        byte[] otherRow = null;
        for (Region region : regions) {
            if (!region.getRegionInfo().getEncodedName().equals(toMove.getRegionInfo().getEncodedName()) && Bytes.BYTES_COMPARATOR.compare(region.getRegionInfo().getStartKey(), ROW_X) < 0) {
                otherRow = region.getRegionInfo().getStartKey();
                break;
            }
        }
        assertNotNull(otherRow);
        // If empty row, set it to first row.-f
        if (otherRow.length <= 0)
            otherRow = Bytes.toBytes("aaa");
        Put put2 = new Put(otherRow);
        put2.addColumn(FAM_NAM, otherRow, otherRow);
        //cache put2's location
        table.put(put2);
        // Check that we are in the expected state
        Assert.assertTrue(curServer != destServer);
        Assert.assertNotEquals(curServer.getServerName(), destServer.getServerName());
        Assert.assertNotEquals(toMove.getPort(), destServerName.getPort());
        Assert.assertNotNull(curServer.getOnlineRegion(regionName));
        Assert.assertNull(destServer.getOnlineRegion(regionName));
        Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates().isRegionsInTransition());
        // Moving. It's possible that we don't have all the regions online at this point, so
        //  the test must depends only on the region we're looking at.
        LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
        TEST_UTIL.getAdmin().move(toMove.getRegionInfo().getEncodedNameAsBytes(), destServerName.getServerName().getBytes());
        while (destServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
            // wait for the move to be finished
            Thread.sleep(1);
        }
        LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
        // Check our new state.
        Assert.assertNull(curServer.getOnlineRegion(regionName));
        Assert.assertNotNull(destServer.getOnlineRegion(regionName));
        Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
        Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
        // Cache was NOT updated and points to the wrong server
        Assert.assertFalse(conn.getCachedLocation(TABLE_NAME3, ROW_X).getRegionLocation().getPort() == destServerName.getPort());
        // Hijack the number of retry to fail after 2 tries
        final int prevNumRetriesVal = setNumTries(conn, 2);
        Put put3 = new Put(ROW_X);
        put3.addColumn(FAM_NAM, ROW_X, ROW_X);
        Put put4 = new Put(otherRow);
        put4.addColumn(FAM_NAM, otherRow, otherRow);
        // do multi
        ArrayList<Put> actions = Lists.newArrayList(put4, put3);
        // first should be a valid row,
        table.batch(actions, null);
        // second we get RegionMovedException.
        setNumTries(conn, prevNumRetriesVal);
    } finally {
        table.close();
    }
}

Example 64 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestMasterProcedureWalLease method testWALfencing.

public void testWALfencing(boolean walRolls) throws IOException {
    final ProcedureStore procStore = getMasterProcedureExecutor().getStore();
    assertTrue("expected WALStore for this test", procStore instanceof WALProcedureStore);
    HMaster firstMaster = UTIL.getHBaseCluster().getMaster();
    // cause WAL rolling after a delete in WAL:
    firstMaster.getConfiguration().setLong(WALProcedureStore.ROLL_THRESHOLD_CONF_KEY, 1);
    HMaster backupMaster3 = Mockito.mock(HMaster.class);
    Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
    Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
    final WALProcedureStore procStore2 = new WALProcedureStore(firstMaster.getConfiguration(), firstMaster.getMasterFileSystem().getFileSystem(), ((WALProcedureStore) procStore).getWALDir(), new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
    // start a second store which should fence the first one out
    LOG.info("Starting new WALProcedureStore");
    procStore2.start(1);
    procStore2.recoverLease();
    // to delete the old WAL files).
    if (walRolls) {
        LOG.info("Inserting into second WALProcedureStore, causing WAL rolls");
        for (int i = 0; i < 512; i++) {
            // insert something to the second store then delete it, causing a WAL roll(s)
            Procedure proc2 = new TestProcedure(i);
            procStore2.insert(proc2, null);
            // delete the procedure so that the WAL is removed later
            procStore2.delete(proc2.getProcId());
        }
    }
    // Now, insert something to the first store, should fail.
    // If the store does a WAL roll and continue with another logId without checking higher logIds
    // it will incorrectly succeed.
    LOG.info("Inserting into first WALProcedureStore");
    try {
        procStore.insert(new TestProcedure(11), null);
        fail("Inserting into Procedure Store should have failed");
    } catch (Exception ex) {
        LOG.info("Received expected exception", ex);
    }
}

Example 65 with HMaster

use of org.apache.hadoop.hbase.master.HMaster in project hbase by apache.

the class TestMasterProcedureWalLease method testWalRecoverLease.

@Test
public void testWalRecoverLease() throws Exception {
    final ProcedureStore masterStore = getMasterProcedureExecutor().getStore();
    assertTrue("expected WALStore for this test", masterStore instanceof WALProcedureStore);
    HMaster firstMaster = UTIL.getHBaseCluster().getMaster();
    // Abort Latch for the master store
    final CountDownLatch masterStoreAbort = new CountDownLatch(1);
    masterStore.registerListener(new ProcedureStore.ProcedureStoreListener() {

        @Override
        public void postSync() {
        }

        @Override
        public void abortProcess() {
            LOG.debug("Abort store of Master");
            masterStoreAbort.countDown();
        }
    });
    // startup a fake master the new WAL store will take the lease
    // and the active master should abort.
    HMaster backupMaster3 = Mockito.mock(HMaster.class);
    Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration();
    Mockito.doReturn(true).when(backupMaster3).isActiveMaster();
    final WALProcedureStore backupStore3 = new WALProcedureStore(firstMaster.getConfiguration(), firstMaster.getMasterFileSystem().getFileSystem(), ((WALProcedureStore) masterStore).getWALDir(), new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3));
    // Abort Latch for the test store
    final CountDownLatch backupStore3Abort = new CountDownLatch(1);
    backupStore3.registerListener(new ProcedureStore.ProcedureStoreListener() {

        @Override
        public void postSync() {
        }

        @Override
        public void abortProcess() {
            LOG.debug("Abort store of backupMaster3");
            backupStore3Abort.countDown();
            backupStore3.stop(true);
        }
    });
    backupStore3.start(1);
    backupStore3.recoverLease();
    // Try to trigger a command on the master (WAL lease expired on the active one)
    HTableDescriptor htd = MasterProcedureTestingUtility.createHTD(TableName.valueOf(name.getMethodName()), "f");
    HRegionInfo[] regions = ModifyRegionUtils.createHRegionInfos(htd, null);
    LOG.debug("submit proc");
    try {
        getMasterProcedureExecutor().submitProcedure(new CreateTableProcedure(getMasterProcedureExecutor().getEnvironment(), htd, regions));
        fail("expected RuntimeException 'sync aborted'");
    } catch (RuntimeException e) {
        LOG.info("got " + e.getMessage());
    }
    LOG.debug("wait master store abort");
    masterStoreAbort.await();
    // Now the real backup master should start up
    LOG.debug("wait backup master to startup");
    MasterProcedureTestingUtility.waitBackupMaster(UTIL, firstMaster);
    assertEquals(true, firstMaster.isStopped());
    // wait the store in here to abort (the test will fail due to timeout if it doesn't)
    LOG.debug("wait the store to abort");
    backupStore3.getStoreTracker().setDeleted(1, false);
    try {
        backupStore3.delete(1);
        fail("expected RuntimeException 'sync aborted'");
    } catch (RuntimeException e) {
        LOG.info("got " + e.getMessage());
    }
    backupStore3Abort.await();
}