Examples with RegionLocations org.apache.hadoop.hbase.RegionLocations used on opensource projects

Example 41 with RegionLocations

use of org.apache.hadoop.hbase.RegionLocations in project hbase by apache.

the class TestHCM method testRegionCaching.

/**
   * Test that when we delete a location using the first row of a region
   * that we really delete it.
   * @throws Exception
   */
@Test
public void testRegionCaching() throws Exception {
    TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAM_NAM).close();
    Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
    // test with no retry, or client cache will get updated after the first failure
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);
    Connection connection = ConnectionFactory.createConnection(conf);
    final Table table = connection.getTable(TABLE_NAME);
    TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
    Put put = new Put(ROW);
    put.addColumn(FAM_NAM, ROW, ROW);
    table.put(put);
    ConnectionImplementation conn = (ConnectionImplementation) connection;
    assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
    final int nextPort = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() + 1;
    HRegionLocation loc = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
    conn.updateCachedLocation(loc.getRegionInfo(), loc.getServerName(), ServerName.valueOf("127.0.0.1", nextPort, HConstants.LATEST_TIMESTAMP), HConstants.LATEST_TIMESTAMP);
    Assert.assertEquals(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort(), nextPort);
    conn.clearRegionCache(TABLE_NAME, ROW.clone());
    RegionLocations rl = conn.getCachedLocation(TABLE_NAME, ROW);
    assertNull("What is this location?? " + rl, rl);
    // We're now going to move the region and check that it works for the client
    // First a new put to add the location in the cache
    conn.clearRegionCache(TABLE_NAME);
    Assert.assertEquals(0, conn.getNumberOfCachedRegionLocations(TABLE_NAME));
    Put put2 = new Put(ROW);
    put2.addColumn(FAM_NAM, ROW, ROW);
    table.put(put2);
    assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
    assertNotNull(conn.getCachedLocation(TableName.valueOf(TABLE_NAME.getName()), ROW.clone()));
    TEST_UTIL.getAdmin().setBalancerRunning(false, false);
    HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
    // We can wait for all regions to be online, that makes log reading easier when debugging
    TEST_UTIL.waitUntilNoRegionsInTransition();
    // Now moving the region to the second server
    HRegionLocation toMove = conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation();
    byte[] regionName = toMove.getRegionInfo().getRegionName();
    byte[] encodedRegionNameBytes = toMove.getRegionInfo().getEncodedNameAsBytes();
    // Choose the other server.
    int curServerId = TEST_UTIL.getHBaseCluster().getServerWith(regionName);
    int destServerId = (curServerId == 0 ? 1 : 0);
    HRegionServer curServer = TEST_UTIL.getHBaseCluster().getRegionServer(curServerId);
    HRegionServer destServer = TEST_UTIL.getHBaseCluster().getRegionServer(destServerId);
    ServerName destServerName = destServer.getServerName();
    // Check that we are in the expected state
    Assert.assertTrue(curServer != destServer);
    Assert.assertFalse(curServer.getServerName().equals(destServer.getServerName()));
    Assert.assertFalse(toMove.getPort() == destServerName.getPort());
    Assert.assertNotNull(curServer.getOnlineRegion(regionName));
    Assert.assertNull(destServer.getOnlineRegion(regionName));
    Assert.assertFalse(TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates().isRegionsInTransition());
    // Moving. It's possible that we don't have all the regions online at this point, so
    //  the test must depends only on the region we're looking at.
    LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
    TEST_UTIL.getAdmin().move(toMove.getRegionInfo().getEncodedNameAsBytes(), destServerName.getServerName().getBytes());
    while (destServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
        // wait for the move to be finished
        Thread.sleep(1);
    }
    LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
    // Check our new state.
    Assert.assertNull(curServer.getOnlineRegion(regionName));
    Assert.assertNotNull(destServer.getOnlineRegion(regionName));
    Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    // Cache was NOT updated and points to the wrong server
    Assert.assertFalse(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() == destServerName.getPort());
    // This part relies on a number of tries equals to 1.
    // We do a put and expect the cache to be updated, even if we don't retry
    LOG.info("Put starting");
    Put put3 = new Put(ROW);
    put3.addColumn(FAM_NAM, ROW, ROW);
    try {
        table.put(put3);
        Assert.fail("Unreachable point");
    } catch (RetriesExhaustedWithDetailsException e) {
        LOG.info("Put done, exception caught: " + e.getClass());
        Assert.assertEquals(1, e.getNumExceptions());
        Assert.assertEquals(1, e.getCauses().size());
        Assert.assertArrayEquals(e.getRow(0).getRow(), ROW);
        // Check that we unserialized the exception as expected
        Throwable cause = ClientExceptionsUtil.findException(e.getCause(0));
        Assert.assertNotNull(cause);
        Assert.assertTrue(cause instanceof RegionMovedException);
    }
    Assert.assertNotNull("Cached connection is null", conn.getCachedLocation(TABLE_NAME, ROW));
    Assert.assertEquals("Previous server was " + curServer.getServerName().getHostAndPort(), destServerName.getPort(), conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
    Assert.assertFalse(destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    Assert.assertFalse(curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes));
    // We move it back to do another test with a scan
    LOG.info("Move starting region=" + toMove.getRegionInfo().getRegionNameAsString());
    TEST_UTIL.getAdmin().move(toMove.getRegionInfo().getEncodedNameAsBytes(), curServer.getServerName().getServerName().getBytes());
    while (curServer.getOnlineRegion(regionName) == null || destServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || curServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameBytes) || master.getAssignmentManager().getRegionStates().isRegionsInTransition()) {
        // wait for the move to be finished
        Thread.sleep(1);
    }
    // Check our new state.
    Assert.assertNotNull(curServer.getOnlineRegion(regionName));
    Assert.assertNull(destServer.getOnlineRegion(regionName));
    LOG.info("Move finished for region=" + toMove.getRegionInfo().getRegionNameAsString());
    // Cache was NOT updated and points to the wrong server
    Assert.assertFalse(conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort() == curServer.getServerName().getPort());
    Scan sc = new Scan();
    sc.setStopRow(ROW);
    sc.setStartRow(ROW);
    // The scanner takes the max retries from the connection configuration, not the table as
    // the put.
    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
    try {
        ResultScanner rs = table.getScanner(sc);
        while (rs.next() != null) {
        }
        Assert.fail("Unreachable point");
    } catch (RetriesExhaustedException e) {
        LOG.info("Scan done, expected exception caught: " + e.getClass());
    }
    // Cache is updated with the right value.
    Assert.assertNotNull(conn.getCachedLocation(TABLE_NAME, ROW));
    Assert.assertEquals("Previous server was " + destServer.getServerName().getHostAndPort(), curServer.getServerName().getPort(), conn.getCachedLocation(TABLE_NAME, ROW).getRegionLocation().getPort());
    TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, RPC_RETRY);
    table.close();
    connection.close();
}

Example 42 with RegionLocations

use of org.apache.hadoop.hbase.RegionLocations in project hbase by apache.

the class TestMetaWithReplicas method testHBaseFsckWithFewerMetaReplicas.

@Test
public void testHBaseFsckWithFewerMetaReplicas() throws Exception {
    ClusterConnection c = (ClusterConnection) ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
    RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false);
    HBaseFsckRepair.closeRegionSilentlyAndWait(c, rl.getRegionLocation(1).getServerName(), rl.getRegionLocation(1).getRegionInfo());
    // check that problem exists
    HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false);
    assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION });
    // fix the problem
    hbck = doFsck(TEST_UTIL.getConfiguration(), true);
    // run hbck again to make sure we don't see any errors
    hbck = doFsck(TEST_UTIL.getConfiguration(), false);
    assertErrors(hbck, new ERROR_CODE[] {});
}

Example 43 with RegionLocations

use of org.apache.hadoop.hbase.RegionLocations in project hbase by apache.

the class TestFromClientSide method testGetStartEndKeysWithRegionReplicas.

@Test
public void testGetStartEndKeysWithRegionReplicas() throws IOException {
    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
    HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
    htd.addFamily(fam);
    byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
    Admin admin = TEST_UTIL.getAdmin();
    admin.createTable(htd, KEYS);
    List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName());
    HRegionLocator locator = (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
    for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
        List<RegionLocations> regionLocations = new ArrayList<>();
        // mock region locations coming from meta with multiple replicas
        for (HRegionInfo region : regions) {
            HRegionLocation[] arr = new HRegionLocation[regionReplication];
            for (int i = 0; i < arr.length; i++) {
                arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
            }
            regionLocations.add(new RegionLocations(arr));
        }
        Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);
        assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
        for (int i = 0; i < KEYS.length + 1; i++) {
            byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
            byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
            assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
            assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
        }
    }
}

Example 44 with RegionLocations

use of org.apache.hadoop.hbase.RegionLocations in project hbase by apache.

the class TestReplicasClient method testLocations.

@Test
public void testLocations() throws Exception {
    byte[] b1 = "testLocations".getBytes();
    openRegion(hriSecondary);
    ClusterConnection hc = (ClusterConnection) HTU.getAdmin().getConnection();
    try {
        hc.clearRegionCache();
        RegionLocations rl = hc.locateRegion(table.getName(), b1, false, false);
        Assert.assertEquals(2, rl.size());
        rl = hc.locateRegion(table.getName(), b1, true, false);
        Assert.assertEquals(2, rl.size());
        hc.clearRegionCache();
        rl = hc.locateRegion(table.getName(), b1, true, false);
        Assert.assertEquals(2, rl.size());
        rl = hc.locateRegion(table.getName(), b1, false, false);
        Assert.assertEquals(2, rl.size());
    } finally {
        closeRegion(hriSecondary);
    }
}

Example 45 with RegionLocations

use of org.apache.hadoop.hbase.RegionLocations in project hbase by apache.

the class TestMasterOperationsForRegionReplicas method testCreateTableWithMultipleReplicas.

@Test
public void testCreateTableWithMultipleReplicas() throws Exception {
    final TableName tableName = TableName.valueOf(name.getMethodName());
    final int numRegions = 3;
    final int numReplica = 2;
    try {
        HTableDescriptor desc = new HTableDescriptor(tableName);
        desc.setRegionReplication(numReplica);
        desc.addFamily(new HColumnDescriptor("family"));
        ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
        TEST_UTIL.waitTableEnabled(tableName);
        validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
        List<HRegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
        assert (hris.size() == numRegions * numReplica);
        // check that the master created expected number of RegionState objects
        for (int i = 0; i < numRegions; i++) {
            for (int j = 0; j < numReplica; j++) {
                HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
                RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionState(replica);
                assert (state != null);
            }
        }
        List<Result> metaRows = MetaTableAccessor.fullScanRegions(ADMIN.getConnection());
        int numRows = 0;
        for (Result result : metaRows) {
            RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
            HRegionInfo hri = locations.getRegionLocation().getRegionInfo();
            if (!hri.getTable().equals(tableName))
                continue;
            numRows += 1;
            HRegionLocation[] servers = locations.getRegionLocations();
            // have two locations for the replicas of a region, and the locations should be different
            assert (servers.length == 2);
            assert (!servers[0].equals(servers[1]));
        }
        assert (numRows == numRegions);
        // The same verification of the meta as above but with the SnapshotOfRegionAssignmentFromMeta
        // class
        validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, ADMIN.getConnection());
        // Now kill the master, restart it and see if the assignments are kept
        ServerName master = TEST_UTIL.getHBaseClusterInterface().getClusterStatus().getMaster();
        TEST_UTIL.getHBaseClusterInterface().stopMaster(master);
        TEST_UTIL.getHBaseClusterInterface().waitForMasterToStop(master, 30000);
        TEST_UTIL.getHBaseClusterInterface().startMaster(master.getHostname(), master.getPort());
        TEST_UTIL.getHBaseClusterInterface().waitForActiveAndReadyMaster();
        for (int i = 0; i < numRegions; i++) {
            for (int j = 0; j < numReplica; j++) {
                HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j);
                RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionState(replica);
                assert (state != null);
            }
        }
        validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, ADMIN.getConnection());
        // Now shut the whole cluster down, and verify the assignments are kept so that the
        // availability constraints are met.
        TEST_UTIL.getConfiguration().setBoolean("hbase.master.startup.retainassign", true);
        TEST_UTIL.shutdownMiniHBaseCluster();
        TEST_UTIL.startMiniHBaseCluster(1, numSlaves);
        TEST_UTIL.waitTableEnabled(tableName);
        validateFromSnapshotFromMeta(TEST_UTIL, tableName, numRegions, numReplica, ADMIN.getConnection());
        // Now shut the whole cluster down, and verify regions are assigned even if there is only
        // one server running
        TEST_UTIL.shutdownMiniHBaseCluster();
        TEST_UTIL.startMiniHBaseCluster(1, 1);
        TEST_UTIL.waitTableEnabled(tableName);
        validateSingleRegionServerAssignment(ADMIN.getConnection(), numRegions, numReplica);
        for (int i = 1; i < numSlaves; i++) {
            //restore the cluster
            TEST_UTIL.getMiniHBaseCluster().startRegionServer();
        }
        //check on alter table
        ADMIN.disableTable(tableName);
        assert (ADMIN.isTableDisabled(tableName));
        //increase the replica
        desc.setRegionReplication(numReplica + 1);
        ADMIN.modifyTable(tableName, desc);
        ADMIN.enableTable(tableName);
        assert (ADMIN.isTableEnabled(tableName));
        List<HRegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
        assert (regions.size() == numRegions * (numReplica + 1));
        //decrease the replica(earlier, table was modified to have a replica count of numReplica + 1)
        ADMIN.disableTable(tableName);
        desc.setRegionReplication(numReplica);
        ADMIN.modifyTable(tableName, desc);
        ADMIN.enableTable(tableName);
        assert (ADMIN.isTableEnabled(tableName));
        regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates().getRegionsOfTable(tableName);
        assert (regions.size() == numRegions * numReplica);
        //also make sure the meta table has the replica locations removed
        hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
        assert (hris.size() == numRegions * numReplica);
        //just check that the number of default replica regions in the meta table are the same
        //as the number of regions the table was created with, and the count of the
        //replicas is numReplica for each region
        Map<HRegionInfo, Integer> defaultReplicas = new HashMap<>();
        for (HRegionInfo hri : hris) {
            Integer i;
            HRegionInfo regionReplica0 = RegionReplicaUtil.getRegionInfoForDefaultReplica(hri);
            defaultReplicas.put(regionReplica0, (i = defaultReplicas.get(regionReplica0)) == null ? 1 : i + 1);
        }
        assert (defaultReplicas.size() == numRegions);
        Collection<Integer> counts = new HashSet<>(defaultReplicas.values());
        assert (counts.size() == 1 && counts.contains(new Integer(numReplica)));
    } finally {
        ADMIN.disableTable(tableName);
        ADMIN.deleteTable(tableName);
    }
}